added notes about cronjob for ha

# Conflicts:
#	README.md

.DEBIAN/conffiles

@@ -1,2 +1 @@
 /etc/fastapi-dls/env
-/etc/systemd/system/fastapi-dls.service

.DEBIAN/control

@@ -2,7 +2,7 @@ Package: fastapi-dls
 Version: 0.0
 Architecture: all
 Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de
-Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl
+Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, python3-httpx, uvicorn, openssl
 Recommends: curl
 Installed-Size: 10240
 Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls

.DEBIAN/postinst

@@ -3,7 +3,7 @@
 WORKING_DIR=/usr/share/fastapi-dls
 CONFIG_DIR=/etc/fastapi-dls
 
-if [[ ! -f $CONFIG_DIR/instance.private.pem ]]; then
+if [ ! -f $CONFIG_DIR/instance.private.pem ]; then
   echo "> Create dls-instance keypair ..."
   openssl genrsa -out $CONFIG_DIR/instance.private.pem 2048
   openssl rsa -in $CONFIG_DIR/instance.private.pem -outform PEM -pubout -out $CONFIG_DIR/instance.public.pem
@@ -12,8 +12,8 @@ else
 fi
 
 while true; do
-  [[ -f $CONFIG_DIR/webserver.key ]] && default_answer="N" || default_answer="Y"
-  [[ $default_answer == "Y" ]] && V="Y/n" || V="y/N"
+  [ -f $CONFIG_DIR/webserver.key ] && default_answer="N" || default_answer="Y"
+  [ $default_answer == "Y" ] && V="Y/n" || V="y/N"
   read -p "> Do you wish to create self-signed webserver certificate? [${V}]" yn
   yn=${yn:-$default_answer} # ${parameter:-word} If parameter is unset or null, the expansion of word is substituted. Otherwise, the value of parameter is substituted.
   case $yn in
@@ -27,7 +27,7 @@ while true; do
   esac
 done
 
-if [[ -f $CONFIG_DIR/webserver.key ]]; then
+if [ -f $CONFIG_DIR/webserver.key ]; then
   echo "> Starting service ..."
   systemctl start fastapi-dls.service
 

.DEBIAN/postrm

@@ -1,8 +1,9 @@
 #!/bin/bash
 
-if [[ -f /etc/systemd/system/fastapi-dls.service ]]; then
-  echo "> Removing service file."
-  rm /etc/systemd/system/fastapi-dls.service
-fi
+# is removed automatically
+#if [ "$1" = purge ] && [ -d /usr/share/fastapi-dls ]; then
+#  echo "> Removing app."
+#  rm -r /usr/share/fastapi-dls
+#fi
 
-# todo
+echo -e "> Done."

.DEBIAN/prerm

@@ -1,5 +1,3 @@
 #!/bin/bash
 
 echo -e "> Starting uninstallation of 'fastapi-dls'!"
-
-# todo

.PKGBUILD/PKGBUILD

@@ -11,6 +11,7 @@ license=('MIT')
 depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastapi' 'python-dotenv' 'python-dateutil' 'python-sqlalchemy' 'python-pycryptodome' 'uvicorn' 'python-markdown' 'openssl')
 provider=("$pkgname")
 install="$pkgname.install"
+backup=('etc/default/fastapi-dls')
 source=('git+file:///builds/oscar.krause/fastapi-dls' # https://gitea.publichub.eu/oscar.krause/fastapi-dls.git
         "$pkgname.default"
         "$pkgname.service"

.gitlab-ci.yml

@@ -1,3 +1,10 @@
+include:
+  - template: Jobs/Code-Quality.gitlab-ci.yml
+  - template: Jobs/Secret-Detection.gitlab-ci.yml
+  - template: Jobs/SAST.gitlab-ci.yml
+  - template: Jobs/Container-Scanning.gitlab-ci.yml
+  - template: Jobs/Dependency-Scanning.gitlab-ci.yml
+
 cache:
   key: one-key-to-rule-them-all
 
@@ -112,10 +119,11 @@ test:
     - openssl rsa -in app/cert/instance.private.pem -outform PEM -pubout -out app/cert/instance.public.pem
     - cd test
   script:
-    - pytest main.py
+    - python -m pytest main.py --junitxml=report.xml
   artifacts:
     reports:
       dotenv: version.env
+      junit: ['**/report.xml']
 
 .test:linux:
   stage: test
@@ -179,6 +187,60 @@ test:archlinux:
     - pacman -Sy
     - pacman -U --noconfirm *.pkg.tar.zst
 
+code_quality:
+  rules:
+    - if: $CODE_QUALITY_DISABLED
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+secret_detection:
+  rules:
+    - if: $SECRET_DETECTION_DISABLED
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+semgrep-sast:
+  rules:
+    - if: $SAST_DISABLED
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+test_coverage:
+  extends: test
+  allow_failure: true
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - pip install pytest pytest-cov
+    - coverage run -m pytest main.py
+    - coverage report
+    - coverage xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: '**/coverage.xml'
+
+container_scanning:
+  dependencies: [ build:docker ]
+  rules:
+    - if: $CONTAINER_SCANNING_DISABLED
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+gemnasium-python-dependency_scanning:
+  rules:
+    - if: $DEPENDENCY_SCANNING_DISABLED
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
 .deploy:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

README.md

@@ -7,11 +7,42 @@ Compatibility tested with official DLS 2.0.1.
 This service can be used without internet connection.
 Only the clients need a connection to this service on configured port.
 
+**Official Links**
+
+- https://git.collinwebdesigns.de/oscar.krause/fastapi-dls
+- https://gitea.publichub.eu/oscar.krause/fastapi-dls
+- Docker Image `collinwebdesigns/fastapi-dls:latest`
+
+*All other repositories are forks! (which  is no bad - just for information and bug reports)*
+
+---
+
 [[_TOC_]]
 
-
 # Setup (Service)
 
+**System requirements**
+
+- 256mb ram
+- 4gb hdd
+
+Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb ram and 750mb hdd.
+
+**Prepare your system**
+
+- Make sure your timezone is set correct on you fastapi-dls server and your client
+
+**HA Setup Notes**
+
+- only *failover mode* is supported by team-green (see *high availability* in official user guide)
+- make sure you're using same configuration on each node
+- use same `instance.private.pem` and `instance.private.key` on each node
+- add `cronjob` on each node with `curl -X GET --insecure https://localhost/-/ha/replicate`
+
+If you want to use *real* HA, you should use a proxy in front of this service and use a clustered database in backend.
+This is not documented and supported by me, but it *can* work. Please ask the community for help.
+Maybe the simplest solution for HA-ing this service is to use a Docker-Swarm with redundant storage and database.
+
 ## Docker
 
 Docker-Images are available here:
@@ -19,6 +50,8 @@ Docker-Images are available here:
 - [Docker-Hub](https://hub.docker.com/repository/docker/collinwebdesigns/fastapi-dls): `collinwebdesigns/fastapi-dls:latest`
 - [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls/main:latest`
 
+The images include database drivers for `postgres`, `mysql`, `mariadb` and `sqlite`.
+
 **Run this on the Docker-Host**
 
 ```shell
@@ -49,10 +82,12 @@ Goto [`docker-compose.yml`](docker-compose.yml) for more advanced example (with
 version: '3.9'
 
 x-dls-variables: &dls-variables
+  TZ: Europe/Berlin # REQUIRED, set your timezone correctly on fastapi-dls AND YOUR CLIENTS !!!
   DLS_URL: localhost # REQUIRED, change to your ip or hostname
   DLS_PORT: 443
-  LEASE_EXPIRE_DAYS: 90
+  LEASE_EXPIRE_DAYS: 90  # 90 days is maximum
   DATABASE: sqlite:////app/database/db.sqlite
+  DEBUG: false
 
 services:
   dls:
@@ -65,7 +100,12 @@ services:
     volumes:
       - /opt/docker/fastapi-dls/cert:/app/cert
       - dls-db:/app/database
-
+    logging:  # optional, for those who do not need logs
+      driver: "json-file"
+      options:
+        max-file: 5
+        max-size: 10m
+            
 volumes:
   dls-db:
 ```
@@ -74,6 +114,8 @@ volumes:
 
 Tested on `Debian 11 (bullseye)`, Ubuntu may also work.
 
+**Make sure you are logged in as root.**
+
 **Install requirements**
 
 ```shell
@@ -98,7 +140,7 @@ chown -R www-data:www-data $WORKING_DIR
 
 ```shell
 WORKING_DIR=/opt/fastapi-dls/app/cert
-mkdir $WORKING_DIR
+mkdir -p $WORKING_DIR
 cd $WORKING_DIR
 # create instance private and public key for singing JWT's
 openssl genrsa -out $WORKING_DIR/instance.private.pem 2048 
@@ -115,11 +157,14 @@ This is only to test whether the service starts successfully.
 ```shell
 cd /opt/fastapi-dls/app
 su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fastapi-dls/app"
+# or
+sudo -u www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fastapi-dls/app"
 ```
 
 **Create config file**
 
 ```shell
+mkdir /etc/fastapi-dls
 cat <<EOF >/etc/fastapi-dls/env
 DLS_URL=127.0.0.1
 DLS_PORT=443
@@ -164,6 +209,108 @@ EOF
 Now you have to run `systemctl daemon-reload`. After that you can start service
 with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`.
 
+## openSUSE Leap (manual method using `git clone` and python virtual environment)
+
+Tested on `openSUSE Leap 15.4`, openSUSE Tumbleweed may also work.
+
+**Install requirements**
+
+```shell
+zypper in -y python310 python3-virtualenv python3-pip
+```
+
+**Install FastAPI-DLS**
+
+```shell
+BASE_DIR=/opt/fastapi-dls
+SERVICE_USER=dls
+mkdir -p ${BASE_DIR}
+cd ${BASE_DIR}
+git clone https://git.collinwebdesigns.de/oscar.krause/fastapi-dls .
+python3.10 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+deactivate
+useradd -r ${SERVICE_USER} -M -d /opt/fastapi-dls
+chown -R ${SERVICE_USER} ${BASE_DIR}
+```
+
+**Create keypair and webserver certificate**
+
+```shell
+CERT_DIR=${BASE_DIR}/app/cert
+SERVICE_USER=dls
+mkdir ${CERT_DIR}
+cd ${CERT_DIR}
+# create instance private and public key for singing JWT's
+openssl genrsa -out ${CERT_DIR}/instance.private.pem 2048 
+openssl rsa -in ${CERT_DIR}/instance.private.pem -outform PEM -pubout -out ${CERT_DIR}/instance.public.pem
+# create ssl certificate for integrated webserver (uvicorn) - because clients rely on ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout  ${CERT_DIR}/webserver.key -out ${CERT_DIR}/webserver.crt
+chown -R ${SERVICE_USER} ${CERT_DIR}
+```
+
+**Test Service**
+
+This is only to test whether the service starts successfully.
+
+```shell
+BASE_DIR=/opt/fastapi-dls
+SERVICE_USER=dls
+cd ${BASE_DIR}
+su - ${SERVICE_USER} -c "${BASE_DIR}/venv/bin/uvicorn main:app --app-dir=${BASE_DIR}/app"
+```
+
+**Create config file**
+
+```shell
+BASE_DIR=/opt/fastapi-dls
+cat <<EOF >/etc/fastapi-dls/env
+# Adjust DSL_URL as needed (accessing from LAN won't work with 127.0.0.1)
+DLS_URL=127.0.0.1
+DLS_PORT=443
+LEASE_EXPIRE_DAYS=90
+DATABASE=sqlite:///${BASE_DIR}/app/db.sqlite
+
+EOF
+```
+
+**Create service**
+
+```shell
+BASE_DIR=/opt/fastapi-dls
+SERVICE_USER=dls
+cat <<EOF >/etc/systemd/system/fastapi-dls.service
+[Unit]
+Description=Service for fastapi-dls vGPU licensing service
+After=network.target
+
+[Service]
+User=${SERVICE_USER}
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+WorkingDirectory=${BASE_DIR}/app
+EnvironmentFile=/etc/fastapi-dls/env
+ExecStart=${BASE_DIR}/venv/bin/uvicorn main:app \\
+  --env-file /etc/fastapi-dls/env \\
+  --host \$DLS_URL --port \$DLS_PORT \\
+  --app-dir ${BASE_DIR}/app \\
+  --ssl-keyfile ${BASE_DIR}/app/cert/webserver.key \\
+  --ssl-certfile ${BASE_DIR}/app/cert/webserver.crt \\
+  --proxy-headers
+Restart=always
+KillSignal=SIGQUIT
+Type=simple
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+```
+
+Now you have to run `systemctl daemon-reload`. After that you can start service
+with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`.
+
 ## Debian/Ubuntu (using `dpkg`)
 
 Packages are available here:
@@ -206,13 +353,17 @@ Packages are available here:
 ```shell
 pacman -Sy
 FILENAME=/opt/fastapi-dls.pkg.tar.zst
-url -o $FILENAME <download-url>
+
+curl -o $FILENAME <download-url>
+# or
+wget -O $FILENAME <download-url>
+
 pacman -U --noconfirm fastapi-dls.pkg.tar.zst
 ```
 
 Start with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`.
 
-## Let's Encrypt Certificate
+## Let's Encrypt Certificate (optional)
 
 If you're using installation via docker, you can use `traefik`. Please refer to their documentation.
 
@@ -230,29 +381,34 @@ After first success you have to replace `--issue` with `--renew`.
 
 # Configuration
 
-| Variable               | Default                                | Usage                                                                                                |
-|------------------------|----------------------------------------|------------------------------------------------------------------------------------------------------|
-| `DEBUG`                | `false`                                | Toggles `fastapi` debug mode                                                                         |
-| `DLS_URL`              | `localhost`                            | Used in client-token to tell guest driver where dls instance is reachable                            |
-| `DLS_PORT`             | `443`                                  | Used in client-token to tell guest driver where dls instance is reachable                            |
-| `TOKEN_EXPIRE_DAYS`    | `1`                                    | Client auth-token validity (used for authenticate client against api, **not `.tok` file!**)          |
-| `LEASE_EXPIRE_DAYS`    | `90`                                   | Lease time in days                                                                                   |
-| `LEASE_RENEWAL_PERIOD` | `0.15`                                 | The percentage of the lease period that must elapse before a licensed client can renew a license \*1 |
-| `DATABASE`             | `sqlite:///db.sqlite`                  | See [official SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html)                  |
-| `CORS_ORIGINS`         | `https://{DLS_URL}`                    | Sets `Access-Control-Allow-Origin` header (comma separated string) \*2                               |
-| `SITE_KEY_XID`         | `00000000-0000-0000-0000-000000000000` | Site identification uuid                                                                             |
-| `INSTANCE_REF`         | `10000000-0000-0000-0000-000000000001` | Instance identification uuid                                                                         |
-| `ALLOTMENT_REF`        | `20000000-0000-0000-0000-000000000001` | Allotment identification uuid                                                                        |
-| `INSTANCE_KEY_RSA`     | `<app-dir>/cert/instance.private.pem`  | Site-wide private RSA key for singing JWTs \*3                                                       |
-| `INSTANCE_KEY_PUB`     | `<app-dir>/cert/instance.public.pem`   | Site-wide public key \*3                                                                             |
+| Variable               | Default                                | Usage                                                                                                              |
+|------------------------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------|
+| `DEBUG`                | `false`                                | Toggles `fastapi` debug mode                                                                                       |
+| `DLS_URL`              | `localhost`                            | Used in client-token to tell guest driver where dls instance is reachable                                          |
+| `DLS_PORT`             | `443`                                  | Used in client-token to tell guest driver where dls instance is reachable                                          |
+| `HA_REPLICATE`         |                                        | `DLS_URL` + `DLS_PORT` of primary DLS instance, e.g. `dls-node:443` (for HA only **two** nodes are supported!) \*1 |
+| `HA_ROLE`              |                                        | `PRIMARY` or `SECONDARY`                                                                                           |
+| `TOKEN_EXPIRE_DAYS`    | `1`                                    | Client auth-token validity (used for authenticate client against api, **not `.tok` file!**)                        |
+| `LEASE_EXPIRE_DAYS`    | `90`                                   | Lease time in days                                                                                                 |
+| `LEASE_RENEWAL_PERIOD` | `0.15`                                 | The percentage of the lease period that must elapse before a licensed client can renew a license \*2               |
+| `DATABASE`             | `sqlite:///db.sqlite`                  | See [official SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html)                                |
+| `CORS_ORIGINS`         | `https://{DLS_URL}`                    | Sets `Access-Control-Allow-Origin` header (comma separated string) \*3                                             |
+| `SITE_KEY_XID`         | `00000000-0000-0000-0000-000000000000` | Site identification uuid                                                                                           |
+| `INSTANCE_REF`         | `10000000-0000-0000-0000-000000000001` | Instance identification uuid                                                                                       |
+| `ALLOTMENT_REF`        | `20000000-0000-0000-0000-000000000001` | Allotment identification uuid                                                                                      |
+| `INSTANCE_KEY_RSA`     | `<app-dir>/cert/instance.private.pem`  | Site-wide private RSA key for singing JWTs \*4                                                                     |
+| `INSTANCE_KEY_PUB`     | `<app-dir>/cert/instance.public.pem`   | Site-wide public key \*4                                                                                           |
 
-\*1 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license
+\*1 If you want to use HA, this value should be point to `secondary` on `primary` and `primary` on `secondary`. Don't
+use same database for both instances!
+
+\*2 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license
 every 4.8 hours. If network connectivity is lost, the loss of connectivity is detected during license renewal and the
 client has 19.2 hours in which to re-establish connectivity before its license expires.
 
-\*2 Always use `https`, since guest-drivers only support secure connections!
+\*3 Always use `https`, since guest-drivers only support secure connections!
 
-\*3 If you recreate instance keys you need to **recreate client-token for each guest**!
+\*4 If you recreate instance keys you need to **recreate client-token for each guest**!
 
 # Setup (Client)
 
@@ -266,26 +422,67 @@ Successfully tested with this package versions:
 
 ## Linux
 
+Download *client-token* and place it into `/etc/nvidia/ClientConfigToken`:
+
+```shell
+curl --insecure -L -X GET https://<dls-hostname-or-ip>/-/client-token -o /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok
+# or
+wget --no-check-certificate -O /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok https://<dls-hostname-or-ip>/-/client-token
+```
+
+Restart `nvidia-gridd` service:
+
 ```shell
-curl --insecure -L -X GET https://<dls-hostname-or-ip>/client-token -o /etc/nvidia/ClientConfigToken/client_configuration_token_$(date '+%d-%m-%Y-%H-%M-%S').tok
 service nvidia-gridd restart
+```
+
+Check licensing status:
+
+```shell
 nvidia-smi -q | grep "License"
 ```
 
-## Windows
+Output should be something like:
 
-Download file and place it into `C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken`.
-Now restart `NvContainerLocalSystem` service.
-
-**Power-Shell**
-
-```Shell
-curl.exe --insecure -L -X GET https://<dls-hostname-or-ip>/client-token -o "C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken\client_configuration_token_$($(Get-Date).tostring('dd-MM-yy-hh-mm-ss')).tok"
-Restart-Service NVDisplay.ContainerLocalSystem
-'C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe' -q | Select-String "License"
+```text
+vGPU Software Licensed Product
+    License Status                    : Licensed (Expiry: YYYY-M-DD hh:mm:ss GMT)
 ```
 
-## Endpoints
+Done. For more information check [troubleshoot section](#troubleshoot).
+
+## Windows
+
+**Power-Shell** (run as administrator!)
+
+Download *client-token* and place it into `C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken`:
+
+```shell
+curl.exe --insecure -L -X GET https://<dls-hostname-or-ip>/-/client-token -o "C:\Program Files\NVIDIA Corporation\vGPU Licensing\ClientConfigToken\client_configuration_token_$($(Get-Date).tostring('dd-MM-yy-hh-mm-ss')).tok"
+```
+
+Restart `NvContainerLocalSystem` service:
+
+```Shell
+Restart-Service NVDisplay.ContainerLocalSystem
+```
+
+Check licensing status:
+
+```shell
+& 'C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe' -q  | Select-String "License"
+```
+
+Output should be something like:
+
+```text
+vGPU Software Licensed Product
+    License Status                    : Licensed (Expiry: YYYY-M-DD hh:mm:ss GMT)
+```
+
+Done. For more information check [troubleshoot section](#troubleshoot).
+
+# Endpoints
 
 ### `GET /`
 
@@ -303,10 +500,6 @@ Shows current runtime environment variables and their values.
 
 HTML rendered README.md.
 
-### `GET /-/docs`, `GET /-/redoc`
-
-OpenAPI specifications rendered from `GET /-/openapi.json`.
-
 ### `GET /-/manage`
 
 Shows a very basic UI to delete origins or leases.
@@ -491,5 +684,4 @@ The error message can safely be ignored (since we have no license limitation :P)
 
 Thanks to vGPU community and all who uses this project and report bugs.
 
-Special thanks to @samicrusader who created build file for ArchLinux.
-
+Special thanks to @samicrusader who created build file for ArchLinux and @cyrus who wrote the section for openSUSE.

app/main.py

@@ -6,10 +6,10 @@ from os.path import join, dirname
 from os import getenv as env
 
 from dotenv import load_dotenv
-from fastapi import FastAPI
+from fastapi import FastAPI, BackgroundTasks
 from fastapi.requests import Request
 from json import loads as json_loads
-from datetime import datetime
+from datetime import datetime, timedelta
 from dateutil.relativedelta import relativedelta
 from calendar import timegm
 from jose import jws, jwk, jwt, JWTError
@@ -19,15 +19,16 @@ from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLRe
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from util import load_key, load_file
+from util import load_key, load_file, ha_replicate
 from orm import Origin, Lease, init as db_init, migrate
 
-logger = logging.getLogger()
 load_dotenv('../version.env')
 
+TZ = datetime.now().astimezone().tzinfo
+
 VERSION, COMMIT, DEBUG = env('VERSION', 'unknown'), env('COMMIT', 'unknown'), bool(env('DEBUG', False))
 
-config = dict(openapi_url='/-/openapi.json', docs_url='/-/docs', redoc_url='/-/redoc')
+config = dict(openapi_url=None, docs_url=None, redoc_url=None)  # dict(openapi_url='/-/openapi.json', docs_url='/-/docs', redoc_url='/-/redoc')
 app = FastAPI(title='FastAPI-DLS', description='Minimal Delegated License Service (DLS).', version=VERSION, **config)
 db = create_engine(str(env('DATABASE', 'sqlite:///db.sqlite')))
 db_init(db), migrate(db)
@@ -35,6 +36,7 @@ db_init(db), migrate(db)
 # everything prefixed with "INSTANCE_*" is used as "SERVICE_INSTANCE_*" or "SI_*" in official dls service
 DLS_URL = str(env('DLS_URL', 'localhost'))
 DLS_PORT = int(env('DLS_PORT', '443'))
+HA_REPLICATE, HA_ROLE = env('HA_REPLICATE', None), env('HA_ROLE', None)  # only failover is supported
 SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000'))
 INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001'))
 ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001'))
@@ -43,6 +45,8 @@ INSTANCE_KEY_PUB = load_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__),
 TOKEN_EXPIRE_DELTA = relativedelta(days=int(env('TOKEN_EXPIRE_DAYS', 1)), hours=int(env('TOKEN_EXPIRE_HOURS', 0)))
 LEASE_EXPIRE_DELTA = relativedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0)))
 LEASE_RENEWAL_PERIOD = float(env('LEASE_RENEWAL_PERIOD', 0.15))
+LEASE_RENEWAL_DELTA = timedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0)))
+CLIENT_TOKEN_EXPIRE_DELTA = relativedelta(years=12)
 CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) else [f'https://{DLS_URL}']
 
 jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256)
@@ -57,6 +61,8 @@ app.add_middleware(
     allow_headers=['*'],
 )
 
+logging.basicConfig()
+logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG if DEBUG else logging.INFO)
 
 
@@ -96,6 +102,7 @@ async def _config():
         'LEASE_EXPIRE_DELTA': str(LEASE_EXPIRE_DELTA),
         'LEASE_RENEWAL_PERIOD': str(LEASE_RENEWAL_PERIOD),
         'CORS_ORIGINS': str(CORS_ORIGINS),
+        'TZ': str(TZ),
     })
 
 
@@ -151,7 +158,8 @@ async def _origins(request: Request, leases: bool = False):
     for origin in session.query(Origin).all():
         x = origin.serialize()
         if leases:
-            x['leases'] = list(map(lambda _: _.serialize(), Lease.find_by_origin_ref(db, origin.origin_ref)))
+            serialize = dict(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA)
+            x['leases'] = list(map(lambda _: _.serialize(**serialize), Lease.find_by_origin_ref(db, origin.origin_ref)))
         response.append(x)
     session.close()
     return JSONr(response)
@@ -168,10 +176,12 @@ async def _leases(request: Request, origin: bool = False):
     session = sessionmaker(bind=db)()
     response = []
     for lease in session.query(Lease).all():
-        x = lease.serialize()
+        serialize = dict(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA)
+        x = lease.serialize(**serialize)
         if origin:
-            # assume that each lease has a valid origin record
-            x['origin'] = session.query(Origin).filter(Origin.origin_ref == lease.origin_ref).first().serialize()
+            lease_origin = session.query(Origin).filter(Origin.origin_ref == lease.origin_ref).first()
+            if lease_origin is not None:
+                x['origin'] = lease_origin.serialize()
         response.append(x)
     session.close()
     return JSONr(response)
@@ -188,7 +198,37 @@ async def _lease_delete(request: Request, lease_ref: str):
 @app.get('/-/client-token', summary='* Client-Token', description='creates a new messenger token for this service instance')
 async def _client_token():
     cur_time = datetime.utcnow()
-    exp_time = cur_time + relativedelta(years=12)
+    exp_time = cur_time + CLIENT_TOKEN_EXPIRE_DELTA
+
+    if HA_REPLICATE is not None and HA_ROLE.lower() == "secondary":
+        return RedirectResponse(f'https://{HA_REPLICATE}/-/client-token')
+
+    idx_port, idx_node = 0, 0
+
+    def create_svc_port_set(port: int):
+        idx = idx_port
+        return {
+            "idx": idx,
+            "d_name": "DLS",
+            "svc_port_map": [{"service": "auth", "port": port}, {"service": "lease", "port": port}]
+        }
+
+    def create_node_url(url: str, svc_port_set_idx: int):
+        idx = idx_node
+        return {"idx": idx, "url": url, "url_qr": url, "svc_port_set_idx": svc_port_set_idx}
+
+    service_instance_configuration = {
+        "nls_service_instance_ref": INSTANCE_REF,
+        "svc_port_set_list": [create_svc_port_set(DLS_PORT)],
+        "node_url_list": [create_node_url(DLS_URL, idx_port)]
+    }
+    idx_port += 1
+    idx_node += 1
+
+    if HA_REPLICATE is not None and HA_ROLE.lower() == "primary":
+        SEC_URL, SEC_PORT, *invalid = HA_REPLICATE.split(':')
+        service_instance_configuration['svc_port_set_list'].append(create_svc_port_set(SEC_PORT))
+        service_instance_configuration['node_url_list'].append(create_node_url(SEC_URL, idx_port))
 
     payload = {
         "jti": str(uuid4()),
@@ -200,17 +240,7 @@ async def _client_token():
         "update_mode": "ABSOLUTE",
         "scope_ref_list": [ALLOTMENT_REF],
         "fulfillment_class_ref_list": [],
-        "service_instance_configuration": {
-            "nls_service_instance_ref": INSTANCE_REF,
-            "svc_port_set_list": [
-                {
-                    "idx": 0,
-                    "d_name": "DLS",
-                    "svc_port_map": [{"service": "auth", "port": DLS_PORT}, {"service": "lease", "port": DLS_PORT}]
-                }
-            ],
-            "node_url_list": [{"idx": 0, "url": DLS_URL, "url_qr": DLS_URL, "svc_port_set_idx": 0}]
-        },
+        "service_instance_configuration": service_instance_configuration,
         "service_instance_public_key_configuration": {
             "service_instance_public_key_me": {
                 "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:],
@@ -230,6 +260,67 @@ async def _client_token():
     return response
 
 
+@app.get('/-/ha/replicate', summary='* HA replicate - trigger')
+async def _ha_replicate_to_ha(request: Request, background_tasks: BackgroundTasks):
+    if HA_REPLICATE is None or HA_ROLE is None:
+        logger.warning('HA replicate endpoint triggerd, but no value for "HA_REPLICATE" or "HA_ROLE" is set!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'no value for "HA_REPLICATE" or "HA_ROLE" set'})
+
+    session = sessionmaker(bind=db)()
+    origins = [origin.serialize() for origin in session.query(Origin).all()]
+    leases = [lease.serialize(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA) for lease in session.query(Lease).all()]
+
+    background_tasks.add_task(ha_replicate, logger, HA_REPLICATE, HA_ROLE, VERSION, DLS_URL, DLS_PORT, SITE_KEY_XID, INSTANCE_REF, origins, leases)
+    return JSONr(status_code=202, content=None)
+
+
+@app.put('/-/ha/replicate', summary='* HA replicate')
+async def _ha_replicate_by_ha(request: Request):
+    j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
+
+    if HA_REPLICATE is None:
+        logger.warning(f'HA replicate endpoint triggerd, but no value for "HA_REPLICATE" is set!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'no value for "HA_REPLICATE" set'})
+
+    version = j.get('VERSION')
+    if version != VERSION:
+        logger.error(f'Version missmatch on HA replication task!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "VERSION"'})
+
+    site_key_xid = j.get('SITE_KEY_XID')
+    if site_key_xid != SITE_KEY_XID:
+        logger.error(f'Site-Key missmatch on HA replication task!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "SITE_KEY_XID"'})
+
+    instance_ref = j.get('INSTANCE_REF')
+    if instance_ref != INSTANCE_REF:
+        logger.error(f'Version missmatch on HA replication task!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "INSTANCE_REF"'})
+
+    sync_timestamp, max_seconds_behind = datetime.fromisoformat(j.get('sync_timestamp')), 30
+    if sync_timestamp <= cur_time - timedelta(seconds=max_seconds_behind):
+        logger.error(f'Request time more than {max_seconds_behind}s behind!')
+        return JSONr(status_code=503, content={'status': 503, 'detail': 'Request time behind'})
+
+    origins, leases = j.get('origins'), j.get('leases')
+    for origin in origins:
+        origin_ref = origin.get('origin_ref')
+        logging.info(f'> [    ha    ]: origin {origin_ref}')
+        data = Origin.deserialize(origin)
+        Origin.create_or_update(db, data)
+
+    for lease in leases:
+        lease_ref = lease.get('lease_ref')
+        x = Lease.find_by_lease_ref(db, lease_ref)
+        if x is not None and x.lease_updated > sync_timestamp:
+            continue
+        logging.info(f'> [    ha    ]: lease {lease_ref}')
+        data = Lease.deserialize(lease)
+        Lease.create_or_update(db, data)
+
+    return JSONr(status_code=202, content=None)
+
+
 # venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py
 @app.post('/auth/v1/origin', description='find or create an origin')
 async def auth_v1_origin(request: Request):
@@ -525,6 +616,34 @@ async def leasing_v1_lessor_shutdown(request: Request):
     return JSONr(response)
 
 
+@app.on_event('startup')
+async def app_on_startup():
+    logger.info(f'''
+    Using timezone: {str(TZ)}. Make sure this is correct and match your clients!
+    
+    Your clients renew their license every {str(Lease.calculate_renewal(LEASE_RENEWAL_PERIOD, LEASE_RENEWAL_DELTA))}.
+    If the renewal fails, the license is {str(LEASE_RENEWAL_DELTA)} valid.
+    
+    Your client-token file (.tok) is valid for {str(CLIENT_TOKEN_EXPIRE_DELTA)}.
+    ''')
+
+    if HA_REPLICATE is not None and HA_ROLE is not None:
+        from hashlib import sha1
+
+        sha1digest = sha1(INSTANCE_KEY_RSA.export_key()).hexdigest()
+        fingerprint_key = ':'.join(sha1digest[i: i + 2] for i in range(0, len(sha1digest), 2))
+        sha1digest = sha1(INSTANCE_KEY_PUB.export_key()).hexdigest()
+        fingerprint_pub = ':'.join(sha1digest[i: i + 2] for i in range(0, len(sha1digest), 2))
+
+        logger.info(f'''
+    HA mode is enabled. Make sure theses fingerprints matches on all your nodes:
+      - INSTANCE_KEY_RSA: "{str(fingerprint_key)}"
+      - INSTANCE_KEY_PUB: "{str(fingerprint_pub)}"
+    
+    This node ({HA_ROLE}) listens to "https://{DLS_URL}:{DLS_PORT}" and replicates to "https://{HA_REPLICATE}".
+        ''')
+
+
 if __name__ == '__main__':
     import uvicorn
 

app/orm.py

@@ -1,9 +1,9 @@
-import datetime
+from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
 
-from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, update, and_, inspect, text
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, declarative_base
 
 Base = declarative_base()
 
@@ -32,6 +32,16 @@ class Origin(Base):
             'os_version': self.os_version,
         }
 
+    @staticmethod
+    def deserialize(j) -> "Origin":
+        return Origin(
+            origin_ref=j.get('origin_ref'),
+            hostname=j.get('hostname'),
+            guest_driver_version=j.get('guest_driver_version'),
+            os_platform=j.get('os_platform'),
+            os_version=j.get('os_version'),
+        )
+
     @staticmethod
     def create_statement(engine: Engine):
         from sqlalchemy.schema import CreateTable
@@ -56,12 +66,12 @@ class Origin(Base):
         session.close()
 
     @staticmethod
-    def delete(engine: Engine, origins: ["Origin"] = None) -> int:
+    def delete(engine: Engine, origin_refs: [str] = None) -> int:
         session = sessionmaker(bind=engine)()
-        if origins is None:
+        if origin_refs is None:
             deletions = session.query(Origin).delete()
         else:
-            deletions = session.query(Origin).filter(Origin.origin_ref in origins).delete()
+            deletions = session.query(Origin).filter(Origin.origin_ref in origin_refs).delete()
         session.commit()
         session.close()
         return deletions
@@ -81,7 +91,10 @@ class Lease(Base):
     def __repr__(self):
         return f'Lease(origin_ref={self.origin_ref}, lease_ref={self.lease_ref}, expires={self.lease_expires})'
 
-    def serialize(self) -> dict:
+    def serialize(self, renewal_period: float, renewal_delta: timedelta) -> dict:
+        lease_renewal = int(Lease.calculate_renewal(renewal_period, renewal_delta).total_seconds())
+        lease_renewal = self.lease_updated + relativedelta(seconds=lease_renewal)
+
         return {
             'lease_ref': self.lease_ref,
             'origin_ref': self.origin_ref,
@@ -89,8 +102,19 @@ class Lease(Base):
             'lease_created': self.lease_created.isoformat(),
             'lease_expires': self.lease_expires.isoformat(),
             'lease_updated': self.lease_updated.isoformat(),
+            'lease_renewal': lease_renewal.isoformat(),
         }
 
+    @staticmethod
+    def deserialize(j) -> "Lease":
+        return Lease(
+            lease_ref=j.get('lease_ref'),
+            origin_ref=j.get('origin_ref'),
+            lease_created=datetime.fromisoformat(j.get('lease_created')),
+            lease_expires=datetime.fromisoformat(j.get('lease_expires')),
+            lease_updated=datetime.fromisoformat(j.get('lease_updated')),
+        )
+
     @staticmethod
     def create_statement(engine: Engine):
         from sqlalchemy.schema import CreateTable
@@ -133,7 +157,7 @@ class Lease(Base):
         return entity
 
     @staticmethod
-    def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime):
+    def renew(engine: Engine, lease: "Lease", lease_expires: datetime, lease_updated: datetime):
         session = sessionmaker(bind=engine)()
         x = dict(lease_expires=lease_expires, lease_updated=lease_updated)
         session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**x))
@@ -156,6 +180,28 @@ class Lease(Base):
         session.close()
         return deletions
 
+    @staticmethod
+    def calculate_renewal(renewal_period: float, delta: timedelta) -> timedelta:
+        """
+        import datetime
+        LEASE_RENEWAL_PERIOD=0.2  # 20%
+        delta = datetime.timedelta(days=1)
+        renew = delta.total_seconds() * LEASE_RENEWAL_PERIOD
+        renew = datetime.timedelta(seconds=renew)
+        expires = delta - renew  # 19.2
+
+        import datetime
+        LEASE_RENEWAL_PERIOD=0.15  # 15%
+        delta = datetime.timedelta(days=90)
+        renew = delta.total_seconds() * LEASE_RENEWAL_PERIOD
+        renew = datetime.timedelta(seconds=renew)
+        expires = delta - renew  # 76 days, 12:00:00 hours
+
+        """
+        renew = delta.total_seconds() * renewal_period
+        renew = timedelta(seconds=renew)
+        return renew
+
 
 def init(engine: Engine):
     tables = [Origin, Lease]
@@ -163,7 +209,7 @@ def init(engine: Engine):
     session = sessionmaker(bind=engine)()
     for table in tables:
         if not db.dialect.has_table(engine.connect(), table.__tablename__):
-            session.execute(str(table.create_statement(engine)))
+            session.execute(text(str(table.create_statement(engine))))
             session.commit()
     session.close()
 

app/util.py

@@ -26,3 +26,29 @@ def generate_key() -> "RsaKey":
         from Cryptodome.PublicKey.RSA import RsaKey
 
     return RSA.generate(bits=2048)
+
+
+def ha_replicate(logger: "logging.Logger", ha_replicate: str, ha_role: str, version: str, dls_url: str, dls_port: int, site_key_xid: str, instance_ref: str, origins: list, leases: list) -> bool:
+    from datetime import datetime
+    import httpx
+
+    if f'{dls_url}:{dls_port}' == ha_replicate:
+        logger.error(f'Failed to replicate this node ({ha_role}) to "{ha_replicate}": can\'t replicate to itself')
+        return False
+
+    data = {
+        'VERSION': str(version),
+        'HA_REPLICATE': f'{dls_url}:{dls_port}',
+        'SITE_KEY_XID': str(site_key_xid),
+        'INSTANCE_REF': str(instance_ref),
+        'origins': origins,
+        'leases': leases,
+        'sync_timestamp': datetime.utcnow().isoformat(),
+    }
+
+    r = httpx.put(f'https://{ha_replicate}/-/ha/replicate', json=data, verify=False)
+    if r.status_code == 202:
+        logger.info(f'Successfully replicated this node ({ha_role}) to "{ha_replicate}".')
+        return True
+    logger.error(f'Failed to replicate this node ({ha_role}) to "{ha_replicate}": {r.status_code} - {r.content}')
+    return False

docker-compose.yml

@@ -14,6 +14,7 @@ services:
     environment:
       <<: *dls-variables
     volumes:
+      - /etc/timezone:/etc/timezone:ro
       - /opt/docker/fastapi-dls/cert:/app/cert  # instance.private.pem, instance.public.pem
       - db:/app/database
     entrypoint: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--app-dir", "/app", "--proxy-headers"]
@@ -30,6 +31,7 @@ services:
       - "80:80"  # for "/leasing/v1/lessor/shutdown" used by windows guests, can't be changed!
       - "443:443"  # first part must match "DLS_PORT"
     volumes:
+      - /etc/timezone:/etc/timezone:ro
       - /opt/docker/fastapi-dls/cert:/opt/cert
     healthcheck:
       test: ["CMD", "curl", "--insecure", "--fail", "https://localhost/-/health"]

requirements.txt

@@ -1,8 +1,9 @@
-fastapi==0.89.1
+fastapi==0.92.0
 uvicorn[standard]==0.20.0
 python-jose==3.3.0
-pycryptodome==3.16.0
+pycryptodome==3.17
 python-dateutil==2.8.2
-sqlalchemy==1.4.46
+sqlalchemy==2.0.3
 markdown==3.4.1
-python-dotenv==0.21.0
+python-dotenv==0.21.1
+httpx==0.23.3

version.env

@@ -1 +1 @@
-VERSION=1.3.1
+VERSION=1.3.5