From f0fdfafaede34ac2c6cd7bb9068c341f62e3e7ae Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 22 Dec 2022 10:41:07 +0100 Subject: [PATCH 001/102] added basic debian package setup and pipeline --- .gitlab-ci.yml | 44 +++++++++++++++++++++++-- DEBIAN/conffiles | 8 +++++ DEBIAN/control | 9 +++++ DEBIAN/postinst | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ DEBIAN/postrm | 8 +++++ DEBIAN/prerm | 5 +++ 6 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 DEBIAN/conffiles create mode 100644 DEBIAN/control create mode 100644 DEBIAN/postinst create mode 100755 DEBIAN/postrm create mode 100755 DEBIAN/prerm diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c24c913..9d529ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,28 @@ cache: key: one-key-to-rule-them-all -build: +build:debian: + # debian:bullseye-slim + image: debian:bookworm-slim # just to get "python3-jose" working + stage: build + before_script: + - apt-get update -qq && apt-get install -qq -y build-essential + - chmod 0755 -R . + # create build directory for .deb sources + - mkdir build + # copy install instructions + - cp -r DEBIAN build/ + # copy app + - mkdir -p build/usr/share/ + - cp -r app build/usr/share/fastapi-dls + script: + - dpkg -b . build.deb + artifacts: + expire_in: 1 week + paths: + - build.deb + +build:docker: image: docker:dind interruptible: true stage: build @@ -15,10 +36,27 @@ build: - docker build . --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} - docker push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} -test: +test:debian: + image: debian:bookworm-slim stage: test + needs: + - job: build:debian + artifacts: true + before_script: + - apt-get update -qq && apt-get install -qq -y jq # systemd script: - - echo "Nothing to do ..." + # test installation + - apt-get install -q -y ./build.deb --fix-missing + # copy example config from GitLab-CI-Variables + #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env + #- systemctl daemon-reload + #- systemctl enable fastapi-dls.service + #- systemctl start fastapi-dls.service + #- if [ "`curl --insecure -s https://localhost:8000/status | jq .status`" != "up" ]; then exit 2; fi + #- systemctl stop fastapi-dls.service + #- systemctl disable fastapi-dls.service + - apt-get purge -qq -y fastapi-dls + - apt-get autoremove -qq -y && apt-get clean -qq deploy: stage: deploy diff --git a/DEBIAN/conffiles b/DEBIAN/conffiles new file mode 100644 index 0000000..02e3534 --- /dev/null +++ b/DEBIAN/conffiles @@ -0,0 +1,8 @@ +/etc/systemd/system/fastapi-dls.service +/etc/fastapi-dls/env +/etc/fastapi-dls/instance.private.pem +/etc/fastapi-dls/instance.public.pem +/etc/fastapi-dls/webserver.key +/etc/fastapi-dls/webserver.crt + +# todo diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 0000000..b7495d8 --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,9 @@ +Package: fastapi-dls +Version: 0.5 +Architecture: all +Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de +Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, uvicorn, openssl +Recommends: curl +Installed-Size: 10240 +Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls +Description: Minimal Delegated License Service (DLS). diff --git a/DEBIAN/postinst b/DEBIAN/postinst new file mode 100644 index 0000000..dc5ee05 --- /dev/null +++ b/DEBIAN/postinst @@ -0,0 +1,85 @@ +#!/bin/bash + +echo "> Install service ..." +echo </etc/systemd/system/fastapi-dls.service +[Unit] +Description=Service for fastapi-dls +After=network.target + +[Service] +User=www-data +Group=www-data +WorkingDirectory=/usr/share/fastapi-dls +ExecStart=uvicorn \ + --host $DLS_URL --port $DLS_PORT \ + --app-dir /usr/share/fastapi-dls/app \ + --ssl-keyfile /etc/fastapi-dls/webserver.key \ + --ssl-certfile /opt/fastapi-dls/webserver.crt \ + --proxy-headers +EnvironmentFile=/etc/fastapi-dls.env +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all + +[Install] +WantedBy=multi-user.target +EOF + +CONFIG_DIR=/etc/fastapi-dls + +echo "> Create config directory ..." +mkdir -p $CONFIG_DIR + +echo "> Writing default config parameters ..." +touch $CONFIG_DIR/fastapi-dls.env +echo <$CONFIG_DIR +DLS_URL=127.0.0.1 +DLS_PORT=443 +LEASE_EXPIRE_DAYS=90 +DATABASE=sqlite:////usr/share/fastapi-dls/db.sqlite +EOF + +echo "> Create dls-instance keypair ..." +openssl genrsa -out $CONFIG_DIR/instance.private.pem 2048 +openssl rsa -in $CONFIG_DIR/instance.private.pem -outform PEM -pubout -out $CONFIG_DIR/instance.public.pem + +while true; do + read -p "> Do you wish to create self-signed webserver certificate? [y/n]" yn + case $yn in + [Yy]*) + openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout $CONFIG_DIR/webserver.key -out $CONFIG_DIR/webserver.crt + break + ;; + [Nn]*) break ;; + *) echo "Please answer [y] or [n]." ;; + esac +done + +if [[ -f $CONFIG_DIR/webserver.key ]]; then + echo "> Starting service ..." + systemctl start fastapi-dls.service + + if [ -x "$(command -v curl)" ]; then + echo "> Testing API ..." + curl --insecure -X GET https://127.0.0.1/status + else + echo "> Testing API failed, curl not available. Please test manually!" + fi +fi + +cat < Removing config directory." + rm -r /etc/fastapi-dls +fi + +# todo diff --git a/DEBIAN/prerm b/DEBIAN/prerm new file mode 100755 index 0000000..296c995 --- /dev/null +++ b/DEBIAN/prerm @@ -0,0 +1,5 @@ +#!/bin/bash + +echo -e "> Starting uninstallation of 'fastapi-dls'!" + +# todo From 394180652ea8c5e7215307fd78a1525d6ca19319 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 22 Dec 2022 12:57:06 +0100 Subject: [PATCH 002/102] migrated from dataset to sqlalchemy --- app/main.py | 68 ++++++++++------ app/orm.py | 198 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 244 insertions(+), 24 deletions(-) create mode 100644 app/orm.py diff --git a/app/main.py b/app/main.py index f35d3fc..279a12e 100644 --- a/app/main.py +++ b/app/main.py @@ -17,16 +17,18 @@ from jose import jws, jwk, jwt from jose.constants import ALGORITHMS from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse, HTMLResponse -import dataset +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey +from orm import Origin, Lease, Auth + logger = logging.getLogger() load_dotenv('../version.env') VERSION, COMMIT, DEBUG = getenv('VERSION', 'unknown'), getenv('COMMIT', 'unknown'), bool(getenv('DEBUG', False)) - def load_file(filename) -> bytes: with open(filename, 'rb') as file: content = file.read() @@ -45,7 +47,7 @@ __details = dict( version=VERSION, ) -app, db = FastAPI(**__details), dataset.connect(str(getenv('DATABASE', 'sqlite:///db.sqlite'))) +app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) @@ -93,13 +95,17 @@ async def status(request: Request): @app.get('/-/origins') async def _origins(request: Request): - response = list(map(lambda x: jsonable_encoder(x), db['origin'].all())) + session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + response = list(map(lambda x: jsonable_encoder(x), session.query(Origin).all())) + session.close() return JSONResponse(response) @app.get('/-/leases') async def _leases(request: Request): - response = list(map(lambda x: jsonable_encoder(x), db['lease'].all())) + session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + response = list(map(lambda x: jsonable_encoder(x), session.query(Lease).all())) + session.close() return JSONResponse(response) @@ -160,14 +166,14 @@ async def auth_v1_origin(request: Request): origin_ref = j['candidate_origin_ref'] logging.info(f'> [ origin ]: {origin_ref}: {j}') - data = dict( + data = Origin( origin_ref=origin_ref, hostname=j['environment']['hostname'], guest_driver_version=j['environment']['guest_driver_version'], os_platform=j['environment']['os_platform'], os_version=j['environment']['os_version'], ) - db['origin'].upsert(data, ['origin_ref']) + Origin.create_or_update(db, data) cur_time = datetime.utcnow() response = { @@ -208,8 +214,9 @@ async def auth_v1_code(request: Request): auth_code = jws.sign(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256) - db['auth'].delete(origin_ref=origin_ref, expires={'<=': cur_time - delta}) - db['auth'].insert(dict(origin_ref=origin_ref, code_challenge=j['code_challenge'], expires=expires)) + Auth.cleanup(db, origin_ref, cur_time - delta) + data = Auth(origin_ref=origin_ref, code_challenge=j['code_challenge'], expires=expires) + Auth.create(db, data) response = { "auth_code": auth_code, @@ -230,7 +237,10 @@ async def auth_v1_token(request: Request): code_challenge = payload['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ auth ]: {origin_ref} ({code_challenge}): {j}') # validate the code challenge @@ -270,8 +280,10 @@ async def leasing_v1_lessor(request: Request): code_challenge = token['origin_ref'] scope_ref_list = j['scope_ref_list'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ create ]: {origin_ref} ({code_challenge}): create leases for scope_ref_list {scope_ref_list}') cur_time = datetime.utcnow() @@ -292,8 +304,8 @@ async def leasing_v1_lessor(request: Request): } }) - data = dict(origin_ref=origin_ref, lease_ref=scope_ref, lease_created=cur_time, lease_expires=expires) - db['lease'].insert_ignore(data, ['origin_ref', 'lease_ref']) # todo: handle update + data = Lease(origin_ref=origin_ref, lease_ref=scope_ref, lease_created=cur_time, lease_expires=expires) + Lease.create_or_update(db, data) response = { "lease_result_list": lease_result_list, @@ -313,8 +325,11 @@ async def leasing_v1_lessor_lease(request: Request): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - active_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref + active_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) logging.info(f'> [ leases ]: {origin_ref} ({code_challenge}): found {len(active_lease_list)} active leases') cur_time = datetime.utcnow() @@ -334,10 +349,14 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ renew ]: {origin_ref} ({code_challenge}): renew {lease_ref}') - if db['lease'].count(origin_ref=origin_ref, lease_ref=lease_ref) == 0: + entity = Lease.find_by_origin_ref_and_lease_ref(db, origin_ref, lease_ref) + if entity is None: raise HTTPException(status_code=404, detail='requested lease not available') cur_time = datetime.utcnow() @@ -351,8 +370,7 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): "sync_timestamp": cur_time.isoformat(), } - data = dict(origin_ref=origin_ref, lease_ref=lease_ref, lease_expires=expires, lease_last_update=cur_time) - db['lease'].update(data, ['origin_ref', 'lease_ref']) + Lease.renew(db, entity, expires, cur_time) return JSONResponse(response) @@ -363,9 +381,13 @@ async def leasing_v1_lessor_lease_remove(request: Request): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - released_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) - deletions = db['lease'].delete(origin_ref=origin_ref) + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref + + released_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) + deletions = Lease.ceanup(db, origin_ref) logging.info(f'> [ remove ]: {origin_ref} ({code_challenge}): removed {deletions} leases') cur_time = datetime.utcnow() diff --git a/app/orm.py b/app/orm.py new file mode 100644 index 0000000..e253b18 --- /dev/null +++ b/app/orm.py @@ -0,0 +1,198 @@ +import datetime + +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.future import Engine +from sqlalchemy.orm import sessionmaker + +Base = declarative_base() + + +class Origin(Base): + __tablename__ = "origin" + + """ + CREATE TABLE origin ( + id INTEGER NOT NULL, + origin_ref TEXT, + hostname TEXT, + guest_driver_version TEXT, + os_platform TEXT, + os_version TEXT, + PRIMARY KEY (id) + ); + CREATE INDEX ix_origin_0548dd22f20de1bb ON origin (origin_ref); + """ + + """ + 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|ubuntu-grid-server|525.60.13|Ubuntu 20.04|20.04.5 LTS (Focal Fossa) + 2|230b0000-a356-4000-8a2b-0000564c0000|PC-WORKSTATION|527.41|Windows 10 Pro|10.0.19045 + 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|docker-cuda-1|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) + 4|41720000-FA43-4000-9472-0000E8660000|PC-Windows|527.41|Windows 10 Pro|10.0.19045 + 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|docker-cuda-2|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) + """ + + origin_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4 + + hostname = Column(VARCHAR(length=256), nullable=True) + guest_driver_version = Column(VARCHAR(length=10), nullable=True) + os_platform = Column(VARCHAR(length=256), nullable=True) + os_version = Column(VARCHAR(length=256), nullable=True) + + def __repr__(self): + return f'Origin(origin_ref={self.origin_ref}, hostname={self.hostname})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Origin.__table__).compile(engine) + + @staticmethod + def create_or_update(engine: Engine, origin: "Origin"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Origin).filter(Origin.origin_ref == origin.origin_ref).first() + print(entity) + if entity is None: + session.add(origin) + else: + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**origin.values())) + session.flush() + session.close() + + +class Auth(Base): + __tablename__ = "auth" + + """ + CREATE TABLE auth ( + id INTEGER NOT NULL, + origin_ref TEXT, + code_challenge TEXT, + expires DATETIME, + PRIMARY KEY (id) + ); + """ + + """ + 20|B210CF72-FEC7-4440-9499-1156D1ACD13A|p8oeBJPumrosywezCQ6VQvI/J2LZbMRK0s+OfsqzAiI|2022-12-21 05:00:57.467359 + 61|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|9Nnv5FMtV9nF8qYRtCKG5lfF23HGvvNCQvpCh3FUITo|2022-12-22 05:08:40.713022 + 65|230b0000-a356-4000-8a2b-0000564c0000|9PivDr3PYRfcdUgODBR5+gi2ZdAbmPb07yTO05uui4A|2022-12-22 06:22:27.409642 + 66|41720000-FA43-4000-9472-0000E8660000|VnyasehSayRX/2OD3YyP8Xn9nsIBVefZpscnIIj2Rpk|2022-12-22 08:58:04.279664 + 67|41720000-FA43-4000-9472-0000E8660000|uisrxDFKB8KuD+JvtgT1ol5pNm/GKKlhO69u2ntg7z0|2022-12-22 08:59:37.509520 + 68|908B202D-CC43-420F-A2EF-FC092AAE8D38|VtWk7It+k33FxiGjm9rlSgAg1ZigfreFJd/0tt30FgQ|2022-12-22 09:43:56.680163 + """ + + _table_args__ = ( + # this can be db.PrimaryKeyConstraint if you want it to be a primary key + UniqueConstraint('origin_ref', 'code_challenge'), + ) + + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) + code_challenge = Column(VARCHAR(length=43), primary_key=True, nullable=False) + expires = Column(DATETIME(), nullable=False) + + def __repr__(self): + return f'Auth(origin_ref={self.origin_ref}, code_challenge={self.code_challenge}, expires={self.expires})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Auth.__table__).compile(engine) + + @staticmethod + def create(engine: Engine, auth: "Auth"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + session.add(auth) + session.flush() + session.close() + + @staticmethod + def cleanup(engine: Engine, origin_ref: str, older_than: datetime.datetime): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + session.execute(delete(Auth).where(and_(Auth.origin_ref == origin_ref, Auth.expires <= older_than))) + session.close() + + @staticmethod + def find_by_code_challenge(engine: Engine, code_challenge: str) -> "Auth": + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Auth).filter(Auth.code_challenge == code_challenge).first() + session.close() + return entity + + +class Lease(Base): + __tablename__ = "lease" + + """ + CREATE TABLE lease ( + id INTEGER NOT NULL, + origin_ref TEXT, + lease_ref TEXT, + lease_created DATETIME, + lease_expires DATETIME, lease_last_update DATETIME, + PRIMARY KEY (id) + ); + CREATE INDEX ix_lease_11c7d13bfb17f70d ON lease (origin_ref, lease_ref); + """ + + """ + 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|9c4536f9-a216-44c7-a1d3-388a15ee80be|2022-12-20 17:29:07.906668|2022-12-22 04:45:58.138211|2022-12-21 04:45:58.138211 + 2|230b0000-a356-4000-8a2b-0000564c0000|1d95e160-058d-4052-b49f-b85306b4c345|2022-12-20 17:30:25.388389|2022-12-23 06:07:29.913027|2022-12-22 06:07:29.913027 + 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|9e1bca05-e247-4847-9de6-8b9a210b353e|2022-12-20 17:31:40.158003|2022-12-23 09:28:57.379008|2022-12-22 09:28:57.379008 + 4|41720000-FA43-4000-9472-0000E8660000|f2ece7fa-d0c6-4af4-901c-6d3b2c3ecf88|2022-12-20 21:03:33.403711|2022-12-23 08:44:39.998754|2022-12-22 08:44:39.998754 + 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|5455f59b-dd70-45c1-82fa-3fd5fae6c037|2022-12-21 06:05:35.085572|2022-12-23 04:53:41.385178|2022-12-22 04:53:41.385178 + """ + + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) + lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) + lease_created = Column(DATETIME(), nullable=False) + lease_expires = Column(DATETIME(), nullable=False) + lease_updated = Column(DATETIME(), nullable=False) + + def __repr__(self): + return f'Lease(origin_ref={self.origin_ref}, lease_ref={self.lease_ref}, expires={self.lease_expires})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Lease.__table__).compile(engine) + + @staticmethod + def create_or_update(engine: Engine, lease: "Lease"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Lease).filter(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).first() + if entity is None: + session.add(lease) + else: + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + session.flush() + session.close() + + @staticmethod + def find_by_origin_ref(engine: Engine, origin_ref: str) -> ["Lease"]: + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entities = session.query(Lease).filter(Lease.origin_ref == origin_ref).all() + session.close() + return entities + + @staticmethod + def find_by_origin_ref_and_lease_ref(engine: Engine, origin_ref: str, lease_ref: str) -> "Lease": + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Lease).filter(and_(Lease.origin_ref == origin_ref, Lease.lease_ref == lease_ref)).first() + session.close() + return entity + + @staticmethod + def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + lease.lease_expires = lease_expires + lease.lease_updated = lease_updated + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + session.close() + + @staticmethod + def cleanup(engine: Engine, origin_ref: str) -> int: + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) + session.close() + return deletions diff --git a/requirements.txt b/requirements.txt index 413b6d1..cceb6a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ uvicorn[standard]==0.20.0 python-jose==3.3.0 pycryptodome==3.16.0 python-dateutil==2.8.2 -dataset==1.5.2 +sqlalchemy==1.4.45 markdown==3.4.1 python-dotenv==0.21.0 From 3f5e3b16c590364923114d822a0bd3003bfd7fcd Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:35:37 +0100 Subject: [PATCH 003/102] added api tests --- .gitlab-ci.yml | 6 +++++- test/main.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 test/main.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c24c913..70b8e02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,9 +16,13 @@ build: - docker push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} test: + image: python:3.10-alpine stage: test + before_script: + - pip install pytest httpx + - cd test script: - - echo "Nothing to do ..." + - pytest main.py deploy: stage: deploy diff --git a/test/main.py b/test/main.py new file mode 100644 index 0000000..d28d1a4 --- /dev/null +++ b/test/main.py @@ -0,0 +1,23 @@ +from starlette.testclient import TestClient +import importlib.util +import sys + +MODULE, PATH = 'main.app', '../app/main.py' + +spec = importlib.util.spec_from_file_location(MODULE, PATH) +main = importlib.util.module_from_spec(spec) +sys.modules[MODULE] = main +spec.loader.exec_module(main) + +client = TestClient(main.app) + + +def test_index(): + response = client.get('/') + assert response.status_code == 200 + + +def test_status(): + response = client.get('/status') + assert response.status_code == 200 + assert response.json()['status'] == 'up' From 906af9430a95c8a29a3ffe73677a25f8f08a882b Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:36:33 +0100 Subject: [PATCH 004/102] .gitlab-ci.yml - fixed installing dependencies --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70b8e02..b0291b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ test: image: python:3.10-alpine stage: test before_script: + - pip install -r requirements.txt - pip install pytest httpx - cd test script: From d5d156e70e0cff08137c6bfcff4572bb4f7c4f16 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:38:53 +0100 Subject: [PATCH 005/102] .gitlab-ci.yml - create test certificates --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0291b8..05f12ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,8 @@ test: before_script: - pip install -r requirements.txt - pip install pytest httpx - - cd test + - openssl genrsa -out app/instance.private.pem 2048 + - openssl rsa -in app/instance.private.pem -outform PEM -pubout -out app/instance.public.pem script: - pytest main.py From 67ed6108a29db249829fa138c9725d5fd3c6f4d2 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:40:27 +0100 Subject: [PATCH 006/102] .gitlab-ci.yml - changed test image to bullseye --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05f12ef..24e6cc9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ build: - docker push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} test: - image: python:3.10-alpine + image: python:3.10-slim-bullseye stage: test before_script: - pip install -r requirements.txt From 3367977652b2fb56a3163a6db284d287a2390905 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:41:18 +0100 Subject: [PATCH 007/102] .gitlab-ci.yml - fixed cd into test --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24e6cc9..4dd4114 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ test: - pip install pytest httpx - openssl genrsa -out app/instance.private.pem 2048 - openssl rsa -in app/instance.private.pem -outform PEM -pubout -out app/instance.public.pem + - cd test script: - pytest main.py From 2c1c9b63b41b5e512d1461272ea3aa4bf4869bf7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:41:23 +0100 Subject: [PATCH 008/102] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 937eb28..3421248 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ venv/ .idea/ app/*.sqlite* app/cert/*.* +.pytest_cache From a58549a1627693322b27374481fe8970806adbce Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:43:02 +0100 Subject: [PATCH 009/102] .gitlab-ci.yml - fixed test cert path --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dd4114..49e8c5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,8 +21,9 @@ test: before_script: - pip install -r requirements.txt - pip install pytest httpx - - openssl genrsa -out app/instance.private.pem 2048 - - openssl rsa -in app/instance.private.pem -outform PEM -pubout -out app/instance.public.pem + - mkdir -p app/cert + - openssl genrsa -out app/cert/instance.private.pem 2048 + - openssl rsa -in app/cert/instance.private.pem -outform PEM -pubout -out app/cert/instance.public.pem - cd test script: - pytest main.py From 3f71c88d48145fb91853de5ded7d5ab11cd5b442 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 07:48:47 +0100 Subject: [PATCH 010/102] added some test --- test/main.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/main.py b/test/main.py index d28d1a4..aeb52e3 100644 --- a/test/main.py +++ b/test/main.py @@ -21,3 +21,51 @@ def test_status(): response = client.get('/status') assert response.status_code == 200 assert response.json()['status'] == 'up' + + +def test_client_token(): + response = client.get('/client-token') + assert response.status_code == 200 + + +def test_auth_v1_origin(): + payload = { + "registration_pending": False, + "environment": { + "guest_driver_version": "guest_driver_version", + "hostname": "myhost", + "ip_address_list": ["192.168.1.123"], + "os_version": "os_version", + "os_platform": "os_platform", + "fingerprint": {"mac_address_list": ["ff:ff:ff:ff:ff:ff"]}, + "host_driver_version": "host_driver_version" + }, + "update_pending": False, + "candidate_origin_ref": "00112233-4455-6677-8899-aabbccddeeff" + } + response = client.post('/auth/v1/origin', json=payload) + assert response.status_code == 200 + + +def test_auth_v1_code(): + pass + + +def test_auth_v1_token(): + pass + + +def test_leasing_v1_lessor(): + pass + + +def test_leasing_v1_lessor_lease(): + pass + + +def test_leasing_v1_lease_renew(): + pass + + +def test_leasing_v1_lessor_lease_remove(): + pass From d1db441df4d8719f23a51c2261db2ec0bcbe65f0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:16:34 +0100 Subject: [PATCH 011/102] removed Auth --- app/main.py | 2 +- app/orm.py | 60 ----------------------------------------------------- 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/app/main.py b/app/main.py index ae3b934..15f0c35 100644 --- a/app/main.py +++ b/app/main.py @@ -22,7 +22,7 @@ from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey -from orm import Origin, Lease, Auth +from orm import Origin, Lease logger = logging.getLogger() load_dotenv('../version.env') diff --git a/app/orm.py b/app/orm.py index e253b18..caae193 100644 --- a/app/orm.py +++ b/app/orm.py @@ -60,66 +60,6 @@ class Origin(Base): session.close() -class Auth(Base): - __tablename__ = "auth" - - """ - CREATE TABLE auth ( - id INTEGER NOT NULL, - origin_ref TEXT, - code_challenge TEXT, - expires DATETIME, - PRIMARY KEY (id) - ); - """ - - """ - 20|B210CF72-FEC7-4440-9499-1156D1ACD13A|p8oeBJPumrosywezCQ6VQvI/J2LZbMRK0s+OfsqzAiI|2022-12-21 05:00:57.467359 - 61|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|9Nnv5FMtV9nF8qYRtCKG5lfF23HGvvNCQvpCh3FUITo|2022-12-22 05:08:40.713022 - 65|230b0000-a356-4000-8a2b-0000564c0000|9PivDr3PYRfcdUgODBR5+gi2ZdAbmPb07yTO05uui4A|2022-12-22 06:22:27.409642 - 66|41720000-FA43-4000-9472-0000E8660000|VnyasehSayRX/2OD3YyP8Xn9nsIBVefZpscnIIj2Rpk|2022-12-22 08:58:04.279664 - 67|41720000-FA43-4000-9472-0000E8660000|uisrxDFKB8KuD+JvtgT1ol5pNm/GKKlhO69u2ntg7z0|2022-12-22 08:59:37.509520 - 68|908B202D-CC43-420F-A2EF-FC092AAE8D38|VtWk7It+k33FxiGjm9rlSgAg1ZigfreFJd/0tt30FgQ|2022-12-22 09:43:56.680163 - """ - - _table_args__ = ( - # this can be db.PrimaryKeyConstraint if you want it to be a primary key - UniqueConstraint('origin_ref', 'code_challenge'), - ) - - origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) - code_challenge = Column(VARCHAR(length=43), primary_key=True, nullable=False) - expires = Column(DATETIME(), nullable=False) - - def __repr__(self): - return f'Auth(origin_ref={self.origin_ref}, code_challenge={self.code_challenge}, expires={self.expires})' - - @staticmethod - def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable - return CreateTable(Auth.__table__).compile(engine) - - @staticmethod - def create(engine: Engine, auth: "Auth"): - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - session.add(auth) - session.flush() - session.close() - - @staticmethod - def cleanup(engine: Engine, origin_ref: str, older_than: datetime.datetime): - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - session.execute(delete(Auth).where(and_(Auth.origin_ref == origin_ref, Auth.expires <= older_than))) - session.close() - - @staticmethod - def find_by_code_challenge(engine: Engine, code_challenge: str) -> "Auth": - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - entity = session.query(Auth).filter(Auth.code_challenge == code_challenge).first() - session.close() - return entity - - class Lease(Base): __tablename__ = "lease" From e7102c4de655756102d7b76d84de9aebc973a9dc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:16:58 +0100 Subject: [PATCH 012/102] fixed updates --- app/orm.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/orm.py b/app/orm.py index caae193..0492687 100644 --- a/app/orm.py +++ b/app/orm.py @@ -55,7 +55,13 @@ class Origin(Base): if entity is None: session.add(origin) else: - session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**origin.values())) + values = dict( + hostname=origin.hostname, + guest_driver_version=origin.guest_driver_version, + os_platform=origin.os_platform, + os_version=origin.os_version, + ) + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(values)) session.flush() session.close() @@ -104,7 +110,8 @@ class Lease(Base): if entity is None: session.add(lease) else: - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) session.flush() session.close() @@ -125,9 +132,8 @@ class Lease(Base): @staticmethod def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - lease.lease_expires = lease_expires - lease.lease_updated = lease_updated - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) session.close() @staticmethod From 43d5736f37cd87806f7c19c92689d7edbb4f6cfc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:22:21 +0100 Subject: [PATCH 013/102] code styling & removed comments --- app/orm.py | 44 ++------------------------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/app/orm.py b/app/orm.py index 0492687..b7b05b7 100644 --- a/app/orm.py +++ b/app/orm.py @@ -11,27 +11,6 @@ Base = declarative_base() class Origin(Base): __tablename__ = "origin" - """ - CREATE TABLE origin ( - id INTEGER NOT NULL, - origin_ref TEXT, - hostname TEXT, - guest_driver_version TEXT, - os_platform TEXT, - os_version TEXT, - PRIMARY KEY (id) - ); - CREATE INDEX ix_origin_0548dd22f20de1bb ON origin (origin_ref); - """ - - """ - 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|ubuntu-grid-server|525.60.13|Ubuntu 20.04|20.04.5 LTS (Focal Fossa) - 2|230b0000-a356-4000-8a2b-0000564c0000|PC-WORKSTATION|527.41|Windows 10 Pro|10.0.19045 - 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|docker-cuda-1|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) - 4|41720000-FA43-4000-9472-0000E8660000|PC-Windows|527.41|Windows 10 Pro|10.0.19045 - 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|docker-cuda-2|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) - """ - origin_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4 hostname = Column(VARCHAR(length=256), nullable=True) @@ -69,28 +48,9 @@ class Origin(Base): class Lease(Base): __tablename__ = "lease" - """ - CREATE TABLE lease ( - id INTEGER NOT NULL, - origin_ref TEXT, - lease_ref TEXT, - lease_created DATETIME, - lease_expires DATETIME, lease_last_update DATETIME, - PRIMARY KEY (id) - ); - CREATE INDEX ix_lease_11c7d13bfb17f70d ON lease (origin_ref, lease_ref); - """ + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) # uuid4 + lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) # uuid4 - """ - 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|9c4536f9-a216-44c7-a1d3-388a15ee80be|2022-12-20 17:29:07.906668|2022-12-22 04:45:58.138211|2022-12-21 04:45:58.138211 - 2|230b0000-a356-4000-8a2b-0000564c0000|1d95e160-058d-4052-b49f-b85306b4c345|2022-12-20 17:30:25.388389|2022-12-23 06:07:29.913027|2022-12-22 06:07:29.913027 - 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|9e1bca05-e247-4847-9de6-8b9a210b353e|2022-12-20 17:31:40.158003|2022-12-23 09:28:57.379008|2022-12-22 09:28:57.379008 - 4|41720000-FA43-4000-9472-0000E8660000|f2ece7fa-d0c6-4af4-901c-6d3b2c3ecf88|2022-12-20 21:03:33.403711|2022-12-23 08:44:39.998754|2022-12-22 08:44:39.998754 - 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|5455f59b-dd70-45c1-82fa-3fd5fae6c037|2022-12-21 06:05:35.085572|2022-12-23 04:53:41.385178|2022-12-22 04:53:41.385178 - """ - - origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) - lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) lease_created = Column(DATETIME(), nullable=False) lease_expires = Column(DATETIME(), nullable=False) lease_updated = Column(DATETIME(), nullable=False) From 6049048bbf41e001e5d11ec8c2acb4064595bc15 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 11:24:40 +0100 Subject: [PATCH 014/102] fixed test --- .gitlab-ci.yml | 2 ++ test/main.py | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49e8c5f..20ae5f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,8 @@ build: test: image: python:3.10-slim-bullseye stage: test + variables: + DATABASE: sqlite:///../app/db.sqlite before_script: - pip install -r requirements.txt - pip install pytest httpx diff --git a/test/main.py b/test/main.py index aeb52e3..aa9d644 100644 --- a/test/main.py +++ b/test/main.py @@ -1,13 +1,11 @@ from starlette.testclient import TestClient -import importlib.util import sys -MODULE, PATH = 'main.app', '../app/main.py' +# add relative path to use packages as they were in the app/ dir +sys.path.append('../') +sys.path.append('../app') -spec = importlib.util.spec_from_file_location(MODULE, PATH) -main = importlib.util.module_from_spec(spec) -sys.modules[MODULE] = main -spec.loader.exec_module(main) +from app import main client = TestClient(main.app) From f539db5933b7156effede2666fac7e4bbe29df5e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:17:19 +0100 Subject: [PATCH 015/102] implemented db_init --- app/main.py | 3 ++- app/orm.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 15f0c35..5060361 100644 --- a/app/main.py +++ b/app/main.py @@ -22,7 +22,7 @@ from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey -from orm import Origin, Lease +from orm import Origin, Lease, init as db_init logger = logging.getLogger() load_dotenv('../version.env') @@ -48,6 +48,7 @@ __details = dict( ) app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) +db_init(db) TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) diff --git a/app/orm.py b/app/orm.py index b7b05b7..bb806ae 100644 --- a/app/orm.py +++ b/app/orm.py @@ -1,6 +1,6 @@ import datetime -from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete, inspect from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.future import Engine from sqlalchemy.orm import sessionmaker @@ -102,3 +102,13 @@ class Lease(Base): deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) session.close() return deletions + + +def init(engine: Engine): + tables = [Origin, Lease] + db = inspect(engine) + session = sessionmaker(bind=engine)() + for table in tables: + if not db.dialect.has_table(engine.connect(), table.__tablename__): + session.execute(str(table.create_statement(engine))) + session.close() From 838e30458d4dc25a72326a4c8b34e23f029b3c65 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:21:52 +0100 Subject: [PATCH 016/102] code styling --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 5060361..322e4a8 100644 --- a/app/main.py +++ b/app/main.py @@ -96,7 +96,7 @@ async def status(request: Request): @app.get('/-/origins') async def _origins(request: Request): - session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + session = sessionmaker(bind=db)() response = list(map(lambda x: jsonable_encoder(x), session.query(Origin).all())) session.close() return JSONResponse(response) @@ -104,7 +104,7 @@ async def _origins(request: Request): @app.get('/-/leases') async def _leases(request: Request): - session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + session = sessionmaker(bind=db)() response = list(map(lambda x: jsonable_encoder(x), session.query(Lease).all())) session.close() return JSONResponse(response) From 3d5d728d676403ba096f737356bcc17d8a6103df Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:22:06 +0100 Subject: [PATCH 017/102] code styling --- app/main.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/app/main.py b/app/main.py index 08c0310..ea4fc62 100644 --- a/app/main.py +++ b/app/main.py @@ -53,6 +53,7 @@ LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) DLS_URL = str(getenv('DLS_URL', 'localhost')) DLS_PORT = int(getenv('DLS_PORT', '443')) SITE_KEY_XID = getenv('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000') +INSTANCE_REF = '00000000-0000-0000-0000-000000000000' INSTANCE_KEY_RSA = load_key(join(dirname(__file__), 'cert/instance.private.pem')) INSTANCE_KEY_PUB = load_key(join(dirname(__file__), 'cert/instance.public.pem')) @@ -109,15 +110,6 @@ async def client_token(): cur_time = datetime.utcnow() exp_time = cur_time + relativedelta(years=12) - service_instance_public_key_configuration = { - "service_instance_public_key_me": { - "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], - "exp": INSTANCE_KEY_PUB.public_key().e, - }, - "service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), - "key_retention_mode": "LATEST_ONLY" - } - payload = { "jti": str(uuid4()), "iss": "NLS Service Instance", @@ -129,7 +121,7 @@ async def client_token(): "scope_ref_list": [str(uuid4())], "fulfillment_class_ref_list": [], "service_instance_configuration": { - "nls_service_instance_ref": "00000000-0000-0000-0000-000000000000", + "nls_service_instance_ref": INSTANCE_REF, "svc_port_set_list": [ { "idx": 0, @@ -139,7 +131,14 @@ async def client_token(): ], "node_url_list": [{"idx": 0, "url": DLS_URL, "url_qr": DLS_URL, "svc_port_set_idx": 0}] }, - "service_instance_public_key_configuration": service_instance_public_key_configuration, + "service_instance_public_key_configuration": { + "service_instance_public_key_me": { + "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], + "exp": INSTANCE_KEY_PUB.public_key().e, + }, + "service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), + "key_retention_mode": "LATEST_ONLY" + }, } content = jws.sign(payload, key=jwt_encode_key, headers=None, algorithm=ALGORITHMS.RS256) From 332b9b23cd884db32a286752403702a860d7bf92 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:31:23 +0100 Subject: [PATCH 018/102] code styling --- app/main.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/app/main.py b/app/main.py index ea4fc62..ea26f64 100644 --- a/app/main.py +++ b/app/main.py @@ -134,7 +134,7 @@ async def client_token(): "service_instance_public_key_configuration": { "service_instance_public_key_me": { "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], - "exp": INSTANCE_KEY_PUB.public_key().e, + "exp": int(INSTANCE_KEY_PUB.public_key().e), }, "service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), "key_retention_mode": "LATEST_ONLY" @@ -154,7 +154,7 @@ async def client_token(): # {"candidate_origin_ref":"00112233-4455-6677-8899-aabbccddeeff","environment":{"fingerprint":{"mac_address_list":["ff:ff:ff:ff:ff:ff"]},"hostname":"my-hostname","ip_address_list":["192.168.178.123","fe80::","fe80::1%enp6s18"],"guest_driver_version":"510.85.02","os_platform":"Debian GNU/Linux 11 (bullseye) 11","os_version":"11 (bullseye)"},"registration_pending":false,"update_pending":false} @app.post('/auth/v1/origin') async def auth_v1_origin(request: Request): - j = json.loads((await request.body()).decode('utf-8')) + j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() origin_ref = j['candidate_origin_ref'] logging.info(f'> [ origin ]: {origin_ref}: {j}') @@ -168,7 +168,6 @@ async def auth_v1_origin(request: Request): db['origin'].upsert(data, ['origin_ref']) - cur_time = datetime.utcnow() response = { "origin_ref": origin_ref, "environment": j['environment'], @@ -187,12 +186,11 @@ async def auth_v1_origin(request: Request): # {"code_challenge":"...","origin_ref":"00112233-4455-6677-8899-aabbccddeeff"} @app.post('/auth/v1/code') async def auth_v1_code(request: Request): - j = json.loads((await request.body()).decode('utf-8')) + j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() origin_ref = j['origin_ref'] logging.info(f'> [ code ]: {origin_ref}: {j}') - cur_time = datetime.utcnow() delta = relativedelta(minutes=15) expires = cur_time + delta @@ -221,7 +219,7 @@ async def auth_v1_code(request: Request): # {"auth_code":"...","code_verifier":"..."} @app.post('/auth/v1/token') async def auth_v1_token(request: Request): - j = json.loads((await request.body()).decode('utf-8')) + j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() payload = jwt.decode(token=j['auth_code'], key=jwt_decode_key) origin_ref = payload['origin_ref'] @@ -231,7 +229,6 @@ async def auth_v1_token(request: Request): if payload['challenge'] != b64enc(sha256(j['code_verifier'].encode('utf-8')).digest()).rstrip(b'=').decode('utf-8'): raise HTTPException(status_code=401, detail='expected challenge did not match verifier') - cur_time = datetime.utcnow() access_expires_on = cur_time + TOKEN_EXPIRE_DELTA new_payload = { @@ -259,13 +256,12 @@ async def auth_v1_token(request: Request): # {'fulfillment_context': {'fulfillment_class_ref_list': []}, 'lease_proposal_list': [{'license_type_qualifiers': {'count': 1}, 'product': {'name': 'NVIDIA RTX Virtual Workstation'}}], 'proposal_evaluation_mode': 'ALL_OF', 'scope_ref_list': ['00112233-4455-6677-8899-aabbccddeeff']} @app.post('/leasing/v1/lessor') async def leasing_v1_lessor(request: Request): - j, token = json.loads((await request.body()).decode('utf-8')), get_token(request) + j, token, cur_time = json.loads((await request.body()).decode('utf-8')), get_token(request), datetime.utcnow() origin_ref = token['origin_ref'] scope_ref_list = j['scope_ref_list'] logging.info(f'> [ create ]: {origin_ref}: create leases for scope_ref_list {scope_ref_list}') - cur_time = datetime.utcnow() lease_result_list = [] for scope_ref in scope_ref_list: expires = cur_time + LEASE_EXPIRE_DELTA @@ -300,14 +296,13 @@ async def leasing_v1_lessor(request: Request): # venv/lib/python3.9/site-packages/nls_dal_service_instance_dls/schema/service_instance/V1_0_21__product_mapping.sql @app.get('/leasing/v1/lessor/leases') async def leasing_v1_lessor_lease(request: Request): - token = get_token(request) + token, cur_time = get_token(request), datetime.utcnow() origin_ref = token['origin_ref'] active_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) logging.info(f'> [ leases ]: {origin_ref}: found {len(active_lease_list)} active leases') - cur_time = datetime.utcnow() response = { "active_lease_list": active_lease_list, "sync_timestamp": cur_time.isoformat(), @@ -320,7 +315,7 @@ async def leasing_v1_lessor_lease(request: Request): # venv/lib/python3.9/site-packages/nls_core_lease/lease_single.py @app.put('/leasing/v1/lease/{lease_ref}') async def leasing_v1_lease_renew(request: Request, lease_ref: str): - token = get_token(request) + token, cur_time = get_token(request), datetime.utcnow() origin_ref = token['origin_ref'] logging.info(f'> [ renew ]: {origin_ref}: renew {lease_ref}') @@ -328,8 +323,11 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): if db['lease'].count(origin_ref=origin_ref, lease_ref=lease_ref) == 0: raise HTTPException(status_code=404, detail='requested lease not available') - cur_time = datetime.utcnow() expires = cur_time + LEASE_EXPIRE_DELTA + + data = dict(origin_ref=origin_ref, lease_ref=lease_ref, lease_expires=expires, lease_last_update=cur_time) + db['lease'].update(data, ['origin_ref', 'lease_ref']) + response = { "lease_ref": lease_ref, "expires": expires.isoformat(), @@ -339,15 +337,12 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): "sync_timestamp": cur_time.isoformat(), } - data = dict(origin_ref=origin_ref, lease_ref=lease_ref, lease_expires=expires, lease_last_update=cur_time) - db['lease'].update(data, ['origin_ref', 'lease_ref']) - return JSONResponse(response) @app.delete('/leasing/v1/lessor/leases') async def leasing_v1_lessor_lease_remove(request: Request): - token = get_token(request) + token, cur_time = get_token(request), datetime.utcnow() origin_ref = token['origin_ref'] @@ -355,13 +350,13 @@ async def leasing_v1_lessor_lease_remove(request: Request): deletions = db['lease'].delete(origin_ref=origin_ref) logging.info(f'> [ remove ]: {origin_ref}: removed {deletions} leases') - cur_time = datetime.utcnow() response = { "released_lease_list": released_lease_list, "release_failure_list": None, "sync_timestamp": cur_time.isoformat(), "prompts": None } + return JSONResponse(response) From 6b7c70e59af7c02b329a62c07b94d1bae9e7cea1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:42:02 +0100 Subject: [PATCH 019/102] tests improved --- test/main.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/main.py b/test/main.py index aeb52e3..0711de5 100644 --- a/test/main.py +++ b/test/main.py @@ -1,3 +1,6 @@ +from uuid import uuid4 + +from jose import jwt from starlette.testclient import TestClient import importlib.util import sys @@ -11,6 +14,8 @@ spec.loader.exec_module(main) client = TestClient(main.app) +ORIGIN_REF = str(uuid4()) + def test_index(): response = client.get('/') @@ -41,14 +46,25 @@ def test_auth_v1_origin(): "host_driver_version": "host_driver_version" }, "update_pending": False, - "candidate_origin_ref": "00112233-4455-6677-8899-aabbccddeeff" + "candidate_origin_ref": ORIGIN_REF, } + response = client.post('/auth/v1/origin', json=payload) assert response.status_code == 200 + assert response.json()['origin_ref'] == ORIGIN_REF def test_auth_v1_code(): - pass + payload = { + "code_challenge": "0wmaiAMAlTIDyz4Fgt2/j0tXnGv72TYbbLs4ISRCZlY", + "origin_ref": ORIGIN_REF, + } + + response = client.post('/auth/v1/code', json=payload) + assert response.status_code == 200 + + payload = jwt.get_unverified_claims(token=response.json()['auth_code']) + assert payload['origin_ref'] == ORIGIN_REF def test_auth_v1_token(): From 81608fe497ab5c25efd81f0830ff2d93b536f326 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:48:48 +0100 Subject: [PATCH 020/102] merged dev into debian --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d529ae..330345a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,19 @@ build:docker: - docker build . --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} - docker push ${CI_REGISTRY}/${CI_PROJECT_PATH}/${CI_BUILD_REF_NAME}:${CI_BUILD_REF} +test: + image: python:3.10-slim-bullseye + stage: test + before_script: + - pip install -r requirements.txt + - pip install pytest httpx + - mkdir -p app/cert + - openssl genrsa -out app/cert/instance.private.pem 2048 + - openssl rsa -in app/cert/instance.private.pem -outform PEM -pubout -out app/cert/instance.public.pem + - cd test + script: + - pytest main.py + test:debian: image: debian:bookworm-slim stage: test From 843d918e59462f397b0313a26ffed9f424ddbbe3 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 14:08:56 +0100 Subject: [PATCH 021/102] added dependencies --- DEBIAN/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEBIAN/control b/DEBIAN/control index b7495d8..aea841a 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -2,7 +2,7 @@ Package: fastapi-dls Version: 0.5 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de -Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, uvicorn, openssl +Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, uvicorn, openssl Recommends: curl Installed-Size: 10240 Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls From 4e17e6da82acfec816493de5cce01469da6759f9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 14:09:13 +0100 Subject: [PATCH 022/102] main.py fixed pycryptodome import --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 4ae3c6b..c5970d3 100644 --- a/app/main.py +++ b/app/main.py @@ -19,8 +19,8 @@ from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse, HTMLResponse from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from Crypto.PublicKey import RSA -from Crypto.PublicKey.RSA import RsaKey +from Cryptodome.PublicKey import RSA # Crypto | Cryptodome on Debian +from Cryptodome.PublicKey.RSA import RsaKey # Crypto | Cryptodome on Debian from orm import Origin, Lease, init as db_init From 599eaba14afa38568c635b9fafb486705c2e45dd Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 09:19:05 +0100 Subject: [PATCH 023/102] README.md - added supported and tested driver versions --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e729b89..ef842d3 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Generate client token, (see [installation](#installation)). There are some more internal api endpoints for handling authentication and lease process. -# Setup +# Setup (Service) ## Docker @@ -194,10 +194,15 @@ with `systemctl start fastapi-dls.service`. | `DATABASE` | `sqlite:///db.sqlite` | See [official dataset docs](https://dataset.readthedocs.io/en/latest/quickstart.html) | | `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) | -# Installation +# Installation (Client) **The token file has to be copied! It's not enough to C&P file contents, because there can be special characters.** +Successfully tested with this package versions: + +- `14.3` (Linux-Host: `510.108.03`, Linux-Guest: `510.108.03`, Windows-Guest: `513.91`) +- `15.0` (Linux-Host: `525.60.12`, Linux-Guest: `525.60.13`, Windows-Guest: `527.41`) + ## Linux ```shell From df0816832eb93e6bc6781f0014917ff8579e884b Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:04:26 +0100 Subject: [PATCH 024/102] fixed conffiles --- .gitlab-ci.yml | 2 ++ DEBIAN/conffiles | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 650262a..a9310fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,8 @@ build:debian: # copy app - mkdir -p build/usr/share/ - cp -r app build/usr/share/fastapi-dls + # create conf file + - touch build/etc/fastapi-dls/env script: - dpkg -b . build.deb artifacts: diff --git a/DEBIAN/conffiles b/DEBIAN/conffiles index 02e3534..008d731 100644 --- a/DEBIAN/conffiles +++ b/DEBIAN/conffiles @@ -1,8 +1 @@ -/etc/systemd/system/fastapi-dls.service /etc/fastapi-dls/env -/etc/fastapi-dls/instance.private.pem -/etc/fastapi-dls/instance.public.pem -/etc/fastapi-dls/webserver.key -/etc/fastapi-dls/webserver.crt - -# todo From f1eddaa99a45d50060df8e2688f0bfac575a1d39 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:05:52 +0100 Subject: [PATCH 025/102] fixed missing directory --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9310fe..daaa4fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ build:debian: - mkdir -p build/usr/share/ - cp -r app build/usr/share/fastapi-dls # create conf file + - mkdir -p build/etc/fastapi-dls - touch build/etc/fastapi-dls/env script: - dpkg -b . build.deb From 98e98ccd8446a1f4bc8b183c988a9ea0a208db7f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:10:00 +0100 Subject: [PATCH 026/102] chroot into "build" dir --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index daaa4fa..b1f9175 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ build:debian: - mkdir -p build/etc/fastapi-dls - touch build/etc/fastapi-dls/env script: - - dpkg -b . build.deb + - chroot build/ dpkg -b . build.deb artifacts: expire_in: 1 week paths: From 1e84e141df5e126cb970d298d4f384ca36662f34 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:16:04 +0100 Subject: [PATCH 027/102] fixes --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1f9175..1e7e0d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,8 +18,10 @@ build:debian: # create conf file - mkdir -p build/etc/fastapi-dls - touch build/etc/fastapi-dls/env + # cd into "build/" + - cd build/ script: - - chroot build/ dpkg -b . build.deb + - dpkg -b . build.deb artifacts: expire_in: 1 week paths: From 5d48f6b7d53380b152ca4ab187e4ee5927e6d26c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:19:35 +0100 Subject: [PATCH 028/102] .gitlab-ci.yml - fixed artifact path --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e7e0d0..72fc8e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ build:debian: artifacts: expire_in: 1 week paths: - - build.deb + - build/build.deb build:docker: image: docker:dind @@ -66,7 +66,7 @@ test:debian: - apt-get update -qq && apt-get install -qq -y jq # systemd script: # test installation - - apt-get install -q -y ./build.deb --fix-missing + - apt-get install -q -y build/build.deb --fix-missing # copy example config from GitLab-CI-Variables #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env #- systemctl daemon-reload From e2cea7136560b0330084c0fef7a1c54db41ad9d1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:22:03 +0100 Subject: [PATCH 029/102] .gitlab-ci.yml - added some debugging --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72fc8e8..5fdbcb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,6 +65,7 @@ test:debian: before_script: - apt-get update -qq && apt-get install -qq -y jq # systemd script: + - ls -alh # test installation - apt-get install -q -y build/build.deb --fix-missing # copy example config from GitLab-CI-Variables From ab30ad2117e6985b6821d6b82e57dcb2c9e82d1c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:23:51 +0100 Subject: [PATCH 030/102] .gitlab-ci.yml - debugging --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5fdbcb9..b2a928b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,9 +65,9 @@ test:debian: before_script: - apt-get update -qq && apt-get install -qq -y jq # systemd script: - - ls -alh + - ls -alh build/ # test installation - - apt-get install -q -y build/build.deb --fix-missing + - apt-get install -q -y ./build/build.deb --fix-missing # copy example config from GitLab-CI-Variables #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env #- systemctl daemon-reload From 60ec2821e2fb6fa4851023644eb1e94fa2bc3fdb Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:38:26 +0100 Subject: [PATCH 031/102] postinst - add default value --- DEBIAN/postinst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index dc5ee05..99e3e41 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -46,7 +46,8 @@ openssl genrsa -out $CONFIG_DIR/instance.private.pem 2048 openssl rsa -in $CONFIG_DIR/instance.private.pem -outform PEM -pubout -out $CONFIG_DIR/instance.public.pem while true; do - read -p "> Do you wish to create self-signed webserver certificate? [y/n]" yn + read -p "> Do you wish to create self-signed webserver certificate? [Y/n]" yn + yn=${yn:-y} # ${parameter:-word} If parameter is unset or null, the expansion of word is substituted. Otherwise, the value of parameter is substituted. case $yn in [Yy]*) openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout $CONFIG_DIR/webserver.key -out $CONFIG_DIR/webserver.crt From 646cca42f4368900309f689147fde4a649555ff9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 10:38:49 +0100 Subject: [PATCH 032/102] .gitlab-ci.yml - removed some debugging --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2a928b..aaad6f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,7 +65,6 @@ test:debian: before_script: - apt-get update -qq && apt-get install -qq -y jq # systemd script: - - ls -alh build/ # test installation - apt-get install -q -y ./build/build.deb --fix-missing # copy example config from GitLab-CI-Variables From a91e1f7018d4d79e327ac89b9ca4de45fb8d381f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 11:03:53 +0100 Subject: [PATCH 033/102] README.md - added supported package version 14.4 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ef842d3..7d20426 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ with `systemctl start fastapi-dls.service`. Successfully tested with this package versions: - `14.3` (Linux-Host: `510.108.03`, Linux-Guest: `510.108.03`, Windows-Guest: `513.91`) +- `14.4` (Linux-Host: `510.108.03`, Linux-Guest: `510.108.03`, Windows-Guest: `514.08`) - `15.0` (Linux-Host: `525.60.12`, Linux-Guest: `525.60.13`, Windows-Guest: `527.41`) ## Linux From 7c8a113fbdb991187e9a698d085f223f6c9ab9db Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 11:05:11 +0100 Subject: [PATCH 034/102] .gitlab-ci.yml - added "DEBIAN_FRONTEND=noninteractive" for debian test --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aaad6f5..29435c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,6 +59,8 @@ test: test:debian: image: debian:bookworm-slim stage: test + variables: + DEBIAN_FRONTEND: noninteractive needs: - job: build:debian artifacts: true From 52fb18dea0cc1089b863765591579347565467e4 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:21:52 +0100 Subject: [PATCH 035/102] main.py - fixed imports for "Crypto" and "Cryptodome" (on debian) --- app/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index c5970d3..75ec633 100644 --- a/app/main.py +++ b/app/main.py @@ -19,8 +19,14 @@ from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse, HTMLResponse from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from Cryptodome.PublicKey import RSA # Crypto | Cryptodome on Debian -from Cryptodome.PublicKey.RSA import RsaKey # Crypto | Cryptodome on Debian + +try: + # Crypto | Cryptodome on Debian + from Crypto.PublicKey import RSA + from Crypto.PublicKey.RSA import RsaKey +except ModuleNotFoundError: + from Cryptodome.PublicKey import RSA + from Cryptodome.PublicKey.RSA import RsaKey from orm import Origin, Lease, init as db_init From 507ce93718d393bedb9fdab7acc1cd447722fdc7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:32:40 +0100 Subject: [PATCH 036/102] .gitlab-ci.yml - test starting service --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29435c4..f74c967 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,6 +69,14 @@ test:debian: script: # test installation - apt-get install -q -y ./build/build.deb --fix-missing + - uvicorn --host 127.0.0.1 --port 443 \ + --app-dir /usr/share/fastapi-dls/app \ + --ssl-keyfile /etc/fastapi-dls/webserver.key \ + --ssl-certfile /opt/fastapi-dls/webserver.crt \ + --proxy-headers & + - FASTAPI_DLS_PID=$! + - echo "> Started service with pid: $FASTAPI_DLS_PID" + - kill $FASTAPI_DLS_PID # copy example config from GitLab-CI-Variables #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env #- systemctl daemon-reload From 701453b18a8e95e70c6231a91366c0f0a0e115bf Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:35:07 +0100 Subject: [PATCH 037/102] .gitlab-ci.yml - fixes --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f74c967..634f427 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,13 +69,13 @@ test:debian: script: # test installation - apt-get install -q -y ./build/build.deb --fix-missing - - uvicorn --host 127.0.0.1 --port 443 \ - --app-dir /usr/share/fastapi-dls/app \ - --ssl-keyfile /etc/fastapi-dls/webserver.key \ - --ssl-certfile /opt/fastapi-dls/webserver.crt \ + - uvicorn --host 127.0.0.1 --port 443 + --app-dir /usr/share/fastapi-dls/app + --ssl-keyfile /etc/fastapi-dls/webserver.key + --ssl-certfile /opt/fastapi-dls/webserver.crt --proxy-headers & - FASTAPI_DLS_PID=$! - - echo "> Started service with pid: $FASTAPI_DLS_PID" + - echo "Started service with pid $FASTAPI_DLS_PID" - kill $FASTAPI_DLS_PID # copy example config from GitLab-CI-Variables #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env From 4df5f18b67c748253f443d0943813efc093f90ef Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:40:33 +0100 Subject: [PATCH 038/102] .gitlab-ci.yml - improved testing --- .gitlab-ci.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 634f427..eae130a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,10 +65,13 @@ test:debian: - job: build:debian artifacts: true before_script: - - apt-get update -qq && apt-get install -qq -y jq # systemd + - apt-get update -qq && apt-get install -qq -y jq script: # test installation - apt-get install -q -y ./build/build.deb --fix-missing + # copy example config from GitLab-CI-Variables + #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env + # start service in background - uvicorn --host 127.0.0.1 --port 443 --app-dir /usr/share/fastapi-dls/app --ssl-keyfile /etc/fastapi-dls/webserver.key @@ -76,15 +79,10 @@ test:debian: --proxy-headers & - FASTAPI_DLS_PID=$! - echo "Started service with pid $FASTAPI_DLS_PID" + # testing service + - if [ "`curl --insecure -s https://127.0.0.1/status | jq .status`" != "up" ]; then echo "Success"; else "Error"; fi + # cleanup - kill $FASTAPI_DLS_PID - # copy example config from GitLab-CI-Variables - #- cat ${EXAMPLE_CONFIG} > /etc/fastapi-dls/env - #- systemctl daemon-reload - #- systemctl enable fastapi-dls.service - #- systemctl start fastapi-dls.service - #- if [ "`curl --insecure -s https://localhost:8000/status | jq .status`" != "up" ]; then exit 2; fi - #- systemctl stop fastapi-dls.service - #- systemctl disable fastapi-dls.service - apt-get purge -qq -y fastapi-dls - apt-get autoremove -qq -y && apt-get clean -qq From 4c643b18dd3bf276a039a556a2b34a7e71f77cea Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:49:12 +0100 Subject: [PATCH 039/102] .gitlab-ci.yml - implemented deploy stage for debian package --- .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eae130a..69ce698 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -86,7 +86,7 @@ test:debian: - apt-get purge -qq -y fastapi-dls - apt-get autoremove -qq -y && apt-get clean -qq -deploy: +deploy:docker: stage: deploy rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -107,3 +107,35 @@ deploy: - docker build . --tag $PUBLIC_REGISTRY_USER/${CI_PROJECT_NAME}:latest - docker push $PUBLIC_REGISTRY_USER/${CI_PROJECT_NAME}:${VERSION} - docker push $PUBLIC_REGISTRY_USER/${CI_PROJECT_NAME}:latest + +deploy:debian: + image: debian:bookworm-slim + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == "debian" # $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + needs: + - job: build:debian + artifacts: true + before_script: + - apt-get update -qq && apt-get install -qq -y curl + script: + # Naming format: _-_.deb + # Version is the version number of the app being packaged + # Release number is the version number of the *packaging* itself. + # The release number might increment if the package maintainer + # updated the packaging, while the version number of the application + # being packaged did not change. + - BUILD_NAME=build.deb # inherited by build-stage + - PACKAGE_NAME=`dpkg -I build/build.deb | grep "Package:" | awk '{ print $2 }'` + - PACKAGE_VERSION=`dpkg -I build/build.deb | grep "Version:" | awk '{ print $2 }'` + - PACKAGE_ARCH=amd64 + - EXPORT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}-0_${PACKAGE_ARCH}.deb" + - mv ${BUILD_NAME} ${EXPORT_NAME} + - 'echo "PACKAGE_NAME: ${PACKAGE_NAME}"' + - 'echo "PACKAGE_VERSION: ${PACKAGE_VERSION}"' + - 'echo "PACKAGE_ARCH: ${PACKAGE_ARCH}"' + - 'echo "EXPORT_NAME: ${EXPORT_NAME}"' + # https://docs.gitlab.com/14.3/ee/user/packages/debian_repository/index.html + - URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/debian/${EXPORT_NAME}" + - 'echo "URL: ${URL}"' + - 'curl --request PUT --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${EXPORT_NAME} ${URL}' From 751546995d5f791d6941a6c35460e08a9d7dd785 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 12:56:46 +0100 Subject: [PATCH 040/102] .gitlab-ci.yml - fixed artifact upload --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69ce698..26d97f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,9 +125,9 @@ deploy:debian: # The release number might increment if the package maintainer # updated the packaging, while the version number of the application # being packaged did not change. - - BUILD_NAME=build.deb # inherited by build-stage - - PACKAGE_NAME=`dpkg -I build/build.deb | grep "Package:" | awk '{ print $2 }'` - - PACKAGE_VERSION=`dpkg -I build/build.deb | grep "Version:" | awk '{ print $2 }'` + - BUILD_NAME=build/build.deb # inherited by build-stage + - PACKAGE_NAME=`dpkg -I ${BUILD_NAME} | grep "Package:" | awk '{ print $2 }'` + - PACKAGE_VERSION=`dpkg -I ${BUILD_NAME} | grep "Version:" | awk '{ print $2 }'` - PACKAGE_ARCH=amd64 - EXPORT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}-0_${PACKAGE_ARCH}.deb" - mv ${BUILD_NAME} ${EXPORT_NAME} From 9d900c4f5c923f1e3eb6bcda002c8be18b7b2a04 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 13:27:27 +0100 Subject: [PATCH 041/102] .gitlab-ci.yml - create initial debian repo --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26d97f9..5b9c2b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -109,6 +109,7 @@ deploy:docker: - docker push $PUBLIC_REGISTRY_USER/${CI_PROJECT_NAME}:latest deploy:debian: + # doc: https://git.collinwebdesigns.de/help/user/packages/debian_repository/index.md#install-a-package image: debian:bookworm-slim stage: deploy rules: @@ -117,7 +118,10 @@ deploy:debian: - job: build:debian artifacts: true before_script: - - apt-get update -qq && apt-get install -qq -y curl + - apt-get update -qq && apt-get install -qq -y curl lsb-release + # create distribution initial + - CODENAME=`lsb_release -cs` + - 'curl --request POST --header "JOB-TOKEN: $CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/debian_distributions?codename=${CODENAME}"' script: # Naming format: _-_.deb # Version is the version number of the app being packaged From 8f5ff50aaf8fe21b59e33d5069d617e8e5107327 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 13:34:21 +0100 Subject: [PATCH 042/102] .gitlab-ci.yml - dynamically create repo for codename if not exist --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b9c2b1..8efda48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,7 +121,8 @@ deploy:debian: - apt-get update -qq && apt-get install -qq -y curl lsb-release # create distribution initial - CODENAME=`lsb_release -cs` - - 'curl --request POST --header "JOB-TOKEN: $CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/debian_distributions?codename=${CODENAME}"' + # create repo if not exists + - 'if [ "`curl -s -o /dev/null -w "%{http_code}" --header "JOB-TOKEN: $CI_JOB_TOKEN" -s ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/debian_distributions/${CODENAME}/key.asc`" != "200" ]; then curl --request POST --header "JOB-TOKEN: $CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/debian_distributions?codename=${CODENAME}"; fi' script: # Naming format: _-_.deb # Version is the version number of the app being packaged From 6947d928ec70c7deacf660632b37fcea0c436b7d Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 13:45:31 +0100 Subject: [PATCH 043/102] .gitlab-ci.yml - fixed artifact upload with access token --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8efda48..1734ea1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,4 +143,5 @@ deploy:debian: # https://docs.gitlab.com/14.3/ee/user/packages/debian_repository/index.html - URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/debian/${EXPORT_NAME}" - 'echo "URL: ${URL}"' - - 'curl --request PUT --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${EXPORT_NAME} ${URL}' + #- 'curl --request PUT --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${EXPORT_NAME} ${URL}' + - 'curl --request PUT --upload-file ${EXPORT_NAME} https://${PRIVATE_WRITE_PACKAGE_REGISTRY_USER}:${PRIVATE_WRITE_PACKAGE_REGISTRY_TOKEN}@git.collinwebdesigns.de/api/v4/projects/${CI_PROJECT_ID}/packages/debian/${EXPORT_NAME}' From c2e04552f7c9af9ec16f20c4c7700292a3048cf1 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 14:45:03 +0100 Subject: [PATCH 044/102] debian - bump version to 0.6.0 --- DEBIAN/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEBIAN/control b/DEBIAN/control index aea841a..aa81b51 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,5 +1,5 @@ Package: fastapi-dls -Version: 0.5 +Version: 0.6.0 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, uvicorn, openssl From 6f143f219950a2d70c72002edf8b6c44597b64ce Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 14:52:17 +0100 Subject: [PATCH 045/102] .gitlab-ci.yml - fixed filename --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1734ea1..baa1479 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,7 +134,8 @@ deploy:debian: - PACKAGE_NAME=`dpkg -I ${BUILD_NAME} | grep "Package:" | awk '{ print $2 }'` - PACKAGE_VERSION=`dpkg -I ${BUILD_NAME} | grep "Version:" | awk '{ print $2 }'` - PACKAGE_ARCH=amd64 - - EXPORT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}-0_${PACKAGE_ARCH}.deb" + #- EXPORT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}-0_${PACKAGE_ARCH}.deb" + - EXPORT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}_${PACKAGE_ARCH}.deb" - mv ${BUILD_NAME} ${EXPORT_NAME} - 'echo "PACKAGE_NAME: ${PACKAGE_NAME}"' - 'echo "PACKAGE_VERSION: ${PACKAGE_VERSION}"' From 6ddba90cd87076c18e9bb440bb399b4e4b689715 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 15:28:52 +0100 Subject: [PATCH 046/102] README fixed --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7d20426..cee024c 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,8 @@ DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 DATABASE=sqlite:////opt/fastapi-dls/app/db.sqlite -EOF + +EOF ``` **Create service** @@ -177,11 +178,12 @@ NotifyAccess=all [Install] WantedBy=multi-user.target + EOF ``` Now you have to run `systemctl daemon-reload`. After that you can start service -with `systemctl start fastapi-dls.service`. +with `systemctl start fastapi-dls.service` (and enable autostart with `systemctl enable fastapi-dls.service`). # Configuration From 3d6da6fab9e30620838f9c91478f791108fb8c91 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 16:59:35 +0100 Subject: [PATCH 047/102] README - fixed debian installation via git --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cee024c..930ca4d 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,9 @@ volumes: Tested on `Debian 11 (bullseye)`, Ubuntu may also work. +**We are running on port `9443` because we are running service as `www-data`-user and non-root users are not allowed to +use ports below 1024!** + **Install requirements** ```shell @@ -112,6 +115,7 @@ python3 -m venv venv source venv/bin/activate pip install -r requirements.txt deactivate +chown -R www-data:www-data $WORKING_DIR ``` **Create keypair and webserver certificate** @@ -125,18 +129,16 @@ openssl genrsa -out $WORKING_DIR/instance.private.pem 2048 openssl rsa -in $WORKING_DIR/instance.private.pem -outform PEM -pubout -out $WORKING_DIR/instance.public.pem # create ssl certificate for integrated webserver (uvicorn) - because clients rely on ssl openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout $WORKING_DIR/webserver.key -out $WORKING_DIR/webserver.crt +chown -R www-data:www-data $WORKING_DIR ``` **Test Service** +This is only to test whether the service starts successfully. + ```shell cd /opt/fastapi-dls/app -/opt/fastapi-dls/venv/bin/uvicorn main:app \ - --host 127.0.0.1 --port 443 \ - --app-dir /opt/fastapi-dls/app \ - --ssl-keyfile /opt/fastapi-dls/app/cert/webserver.key \ - --ssl-certfile /opt/fastapi-dls/app/cert/webserver.crt \ - --proxy-headers +su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fastapi-dls/app" ``` **Create config file** @@ -144,7 +146,7 @@ cd /opt/fastapi-dls/app ```shell cat < /etc/fastapi-dls.env DLS_URL=127.0.0.1 -DLS_PORT=443 +DLS_PORT=9443 LEASE_EXPIRE_DAYS=90 DATABASE=sqlite:////opt/fastapi-dls/app/db.sqlite @@ -163,13 +165,14 @@ After=network.target User=www-data Group=www-data WorkingDirectory=/opt/fastapi-dls/app -ExecStart=/opt/fastapi-dls/venv/bin/uvicorn \ - --host $DLS_URL --port $DLS_PORT \ +EnvironmentFile=/etc/fastapi-dls.env +ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \ + --env-file /etc/fastapi-dls.env \ + --host \$DLS_URL --port \$DLS_PORT \ --app-dir /opt/fastapi-dls/app \ --ssl-keyfile /opt/fastapi-dls/app/cert/webserver.key \ --ssl-certfile /opt/fastapi-dls/app/cert/webserver.crt \ --proxy-headers -EnvironmentFile=/etc/fastapi-dls.env Restart=always KillSignal=SIGQUIT Type=notify From 7898052207f4274037202c9e436f48283a7a7754 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 17:00:33 +0100 Subject: [PATCH 048/102] fixed service --- DEBIAN/postinst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 99e3e41..1e8d83e 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -10,13 +10,14 @@ After=network.target User=www-data Group=www-data WorkingDirectory=/usr/share/fastapi-dls -ExecStart=uvicorn \ +EnvironmentFile=/etc/fastapi-dls.env +ExecStart=uvicorn main:app \ + --env-file /etc/fastapi-dls.env \ --host $DLS_URL --port $DLS_PORT \ --app-dir /usr/share/fastapi-dls/app \ --ssl-keyfile /etc/fastapi-dls/webserver.key \ --ssl-certfile /opt/fastapi-dls/webserver.crt \ --proxy-headers -EnvironmentFile=/etc/fastapi-dls.env Restart=always KillSignal=SIGQUIT Type=notify From f9e37401506ce95f0b9817c6c4505d7c0358d74a Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 17:42:58 +0100 Subject: [PATCH 049/102] main.py - added env variable for "INSTANCE_REF" --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index ea26f64..4b04b69 100644 --- a/app/main.py +++ b/app/main.py @@ -52,8 +52,8 @@ LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) DLS_URL = str(getenv('DLS_URL', 'localhost')) DLS_PORT = int(getenv('DLS_PORT', '443')) -SITE_KEY_XID = getenv('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000') -INSTANCE_REF = '00000000-0000-0000-0000-000000000000' +SITE_KEY_XID = str(getenv('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) +INSTANCE_REF = str(getenv('INSTANCE_REF', '00000000-0000-0000-0000-000000000000')) INSTANCE_KEY_RSA = load_key(join(dirname(__file__), 'cert/instance.private.pem')) INSTANCE_KEY_PUB = load_key(join(dirname(__file__), 'cert/instance.public.pem')) From e5f557eb964dadc2b03c5eb63c655dba2c6c588c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 17:49:52 +0100 Subject: [PATCH 050/102] README.md - added todos --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 930ca4d..3717a93 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,13 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. +## ToDo#'s + +- provide `.deb` package (WIP) +- migrate from `dataset` to `sqlalchemy` (WIP) +- migrate from `fastapi` to `flask` +- Support http mode for using external https proxy + ## Endpoints ### `GET /` From cefee22202f9d61931d4c91fab229d4628dbf978 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 18:38:26 +0100 Subject: [PATCH 051/102] README.md - fixed srevice type --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3717a93..6c79e9b 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \ --proxy-headers Restart=always KillSignal=SIGQUIT -Type=notify +Type=simple StandardError=syslog NotifyAccess=all From 11a2c1d12916d2528cd9e4ec8d684ded8615a80e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 18:51:20 +0100 Subject: [PATCH 052/102] added "CAP_NET_BIND_SERVICE" to debian service to allow low range ports for non root user "www-data" --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6c79e9b..6b014e5 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,6 @@ volumes: Tested on `Debian 11 (bullseye)`, Ubuntu may also work. -**We are running on port `9443` because we are running service as `www-data`-user and non-root users are not allowed to -use ports below 1024!** - **Install requirements** ```shell @@ -153,7 +150,7 @@ su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fast ```shell cat < /etc/fastapi-dls.env DLS_URL=127.0.0.1 -DLS_PORT=9443 +DLS_PORT=443 LEASE_EXPIRE_DAYS=90 DATABASE=sqlite:////opt/fastapi-dls/app/db.sqlite @@ -171,6 +168,7 @@ After=network.target [Service] User=www-data Group=www-data +AmbientCapabilities=CAP_NET_BIND_SERVICE WorkingDirectory=/opt/fastapi-dls/app EnvironmentFile=/etc/fastapi-dls.env ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \ From 6d5ed1a14220024525efeeac396aaa5ec7b6e6c9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 19:03:03 +0100 Subject: [PATCH 053/102] main.py - added origin update endpoint --- app/main.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/main.py b/app/main.py index 4b04b69..93a2b0b 100644 --- a/app/main.py +++ b/app/main.py @@ -181,6 +181,33 @@ async def auth_v1_origin(request: Request): return JSONResponse(response) +# venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py +# { "environment" : { "guest_driver_version" : "guest_driver_version", "hostname" : "myhost", "ip_address_list" : [ "192.168.1.129" ], "os_version" : "os_version", "os_platform" : "os_platform", "fingerprint" : { "mac_address_list" : [ "e4:b9:7a:e5:7b:ff" ] }, "host_driver_version" : "host_driver_version" }, "origin_ref" : "00112233-4455-6677-8899-aabbccddeeff" } +@app.post('/auth/v1/origin/update') +async def auth_v1_origin_update(request: Request): + j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() + + origin_ref = j['origin_ref'] + logging.info(f'> [ update ]: {origin_ref}: {j}') + + data = dict( + origin_ref=origin_ref, + hostname=j['environment']['hostname'], + guest_driver_version=j['environment']['guest_driver_version'], + os_platform=j['environment']['os_platform'], os_version=j['environment']['os_version'], + ) + + db['origin'].upsert(data, ['origin_ref']) + + response = { + "environment": j['environment'], + "prompts": None, + "sync_timestamp": cur_time.isoformat() + } + + return JSONResponse(response) + + # venv/lib/python3.9/site-packages/nls_services_auth/test/test_auth_controller.py # venv/lib/python3.9/site-packages/nls_core_auth/auth.py - CodeResponse # {"code_challenge":"...","origin_ref":"00112233-4455-6677-8899-aabbccddeeff"} From b5c64038cbde808d7e17182c6d27a898caa6fa5b Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 19:05:41 +0100 Subject: [PATCH 054/102] main.py - migrated merged changes from dataset to sqlalchemy --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 4704ab1..84c5968 100644 --- a/app/main.py +++ b/app/main.py @@ -197,14 +197,14 @@ async def auth_v1_origin_update(request: Request): origin_ref = j['origin_ref'] logging.info(f'> [ update ]: {origin_ref}: {j}') - data = dict( + data = Origin( origin_ref=origin_ref, hostname=j['environment']['hostname'], guest_driver_version=j['environment']['guest_driver_version'], os_platform=j['environment']['os_platform'], os_version=j['environment']['os_version'], ) - db['origin'].upsert(data, ['origin_ref']) + Origin.create_or_update(db, data) response = { "environment": j['environment'], From 560b18b5c4ff9c2c143a17d2dcf92b0aeadf81db Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 19:57:58 +0100 Subject: [PATCH 055/102] orm.py - fixed not null column --- app/orm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/orm.py b/app/orm.py index bb806ae..7982b84 100644 --- a/app/orm.py +++ b/app/orm.py @@ -68,6 +68,8 @@ class Lease(Base): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() entity = session.query(Lease).filter(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).first() if entity is None: + if lease.lease_updated is None: + lease.lease_updated = lease.lease_created session.add(lease) else: values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) From 07f1e645531ac908a60f07bf51f7c29ca27b873e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:05:55 +0100 Subject: [PATCH 056/102] fixes --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 84c5968..b35db7d 100644 --- a/app/main.py +++ b/app/main.py @@ -334,7 +334,7 @@ async def leasing_v1_lessor_lease(request: Request): origin_ref = token['origin_ref'] - active_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) + active_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) logging.info(f'> [ leases ]: {origin_ref}: found {len(active_lease_list)} active leases') response = { From 85736c5ce4478d6b2efe71dc844bf23485864fc7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:10:18 +0100 Subject: [PATCH 057/102] typos --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index b35db7d..52ec2ca 100644 --- a/app/main.py +++ b/app/main.py @@ -380,7 +380,7 @@ async def leasing_v1_lessor_lease_remove(request: Request): origin_ref = token['origin_ref'] released_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) - deletions = Lease.ceanup(db, origin_ref) + deletions = Lease.cleanup(db, origin_ref) logging.info(f'> [ remove ]: {origin_ref}: removed {deletions} leases') response = { From 2a3e740964d029c9386409c57a7fe3f89eb951e9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:19:23 +0100 Subject: [PATCH 058/102] added toc --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b014e5..37914cc 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. -## ToDo#'s +[[TOC]] + +## ToDo#'s / Roadmap - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) From 12bfd4c82aa82a04e249b290169ab0a121ce2d72 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:19:50 +0100 Subject: [PATCH 059/102] removed toc --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 37914cc..a61cf80 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,7 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. -[[TOC]] - -## ToDo#'s / Roadmap +## ToDo's - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) From 1b34edfda6596408d6e97ee4062249caca46ec09 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:22:00 +0100 Subject: [PATCH 060/102] fixes --- app/orm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/orm.py b/app/orm.py index 7982b84..31fa64c 100644 --- a/app/orm.py +++ b/app/orm.py @@ -40,7 +40,7 @@ class Origin(Base): os_platform=origin.os_platform, os_version=origin.os_version, ) - session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(values)) + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**values)) session.flush() session.close() @@ -73,7 +73,7 @@ class Lease(Base): session.add(lease) else: values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**values)) session.flush() session.close() @@ -95,7 +95,7 @@ class Lease(Base): def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**values)) session.close() @staticmethod From c38ed25a2fa22573dc0728a8d8682717be310ba6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:28:09 +0100 Subject: [PATCH 061/102] fixes --- app/orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/orm.py b/app/orm.py index 31fa64c..26b869a 100644 --- a/app/orm.py +++ b/app/orm.py @@ -101,7 +101,7 @@ class Lease(Base): @staticmethod def cleanup(engine: Engine, origin_ref: str) -> int: session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) + deletions = session.query(Lease).filter(Lease.origin_ref == origin_ref).delete() session.close() return deletions From 15c49d396ffbbbd07711acb694aef4c9700f9a82 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:35:04 +0100 Subject: [PATCH 062/102] README.md - added required cipher suite for windows guests --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6b014e5..679e026 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,12 @@ Currently, there are no known issues. ## Windows +### Required cipher on Windows Guests (e.g. managed by domain controller with GPO) + +It is required to enable `SHA1` (`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521`) in [windows cipher suite](https://learn.microsoft.com/en-us/windows-server/security/tls/manage-tls). + +### Multiple Display Container LS Instances + On Windows on some machines there are running two or more instances of `NVIDIA Display Container LS`. This causes a problem on licensing flow. As you can see in the logs below, there are two lines with `NLS initialized`, each prefixed with `<1>` and `<2>`. So it is possible, that *daemon 1* fetches a valid license through dls-service, and *daemon 2* From 9a0db3c18f0b4fd4ff1295f621a159d46e058017 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 21:58:08 +0100 Subject: [PATCH 063/102] .gitlab-ci.yml - using generic package registry temporary --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baa1479..e60da24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,4 +145,6 @@ deploy:debian: - URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/debian/${EXPORT_NAME}" - 'echo "URL: ${URL}"' #- 'curl --request PUT --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${EXPORT_NAME} ${URL}' - - 'curl --request PUT --upload-file ${EXPORT_NAME} https://${PRIVATE_WRITE_PACKAGE_REGISTRY_USER}:${PRIVATE_WRITE_PACKAGE_REGISTRY_TOKEN}@git.collinwebdesigns.de/api/v4/projects/${CI_PROJECT_ID}/packages/debian/${EXPORT_NAME}' + # using generic-package-registry until debian-registry is GA + # https://docs.gitlab.com/ee/user/packages/generic_packages/index.html#publish-a-generic-package-by-using-cicd + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${EXPORT_NAME} "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${PACKAGE_VERSION}/${EXPORT_NAME}"' From 18d6da8ebff9c0e6c2e0cb271de4d0ea55c8c35e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 22:18:02 +0100 Subject: [PATCH 064/102] fixes --- DEBIAN/postinst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 1e8d83e..c7d374e 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -26,6 +26,7 @@ NotifyAccess=all [Install] WantedBy=multi-user.target + EOF CONFIG_DIR=/etc/fastapi-dls @@ -35,11 +36,12 @@ mkdir -p $CONFIG_DIR echo "> Writing default config parameters ..." touch $CONFIG_DIR/fastapi-dls.env -echo <$CONFIG_DIR +echo < $CONFIG_DIR/fastapi-dls.env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 DATABASE=sqlite:////usr/share/fastapi-dls/db.sqlite + EOF echo "> Create dls-instance keypair ..." @@ -84,4 +86,5 @@ cat < Date: Wed, 28 Dec 2022 06:46:42 +0100 Subject: [PATCH 065/102] postinst - fixed "cat" instead of "echo" --- DEBIAN/postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index c7d374e..7562e31 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -1,7 +1,7 @@ #!/bin/bash echo "> Install service ..." -echo </etc/systemd/system/fastapi-dls.service +cat < /etc/systemd/system/fastapi-dls.service [Unit] Description=Service for fastapi-dls After=network.target @@ -36,7 +36,7 @@ mkdir -p $CONFIG_DIR echo "> Writing default config parameters ..." touch $CONFIG_DIR/fastapi-dls.env -echo < $CONFIG_DIR/fastapi-dls.env +cat < $CONFIG_DIR/fastapi-dls.env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 From 548e1c94926df108385850142b7d1b40191db30f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:47:06 +0100 Subject: [PATCH 066/102] postinst - fixed service file --- DEBIAN/postinst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 7562e31..ea81cfa 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -9,18 +9,19 @@ After=network.target [Service] User=www-data Group=www-data +AmbientCapabilities=CAP_NET_BIND_SERVICE +EnvironmentFile=/etc/fastapi-dls/env WorkingDirectory=/usr/share/fastapi-dls -EnvironmentFile=/etc/fastapi-dls.env -ExecStart=uvicorn main:app \ - --env-file /etc/fastapi-dls.env \ - --host $DLS_URL --port $DLS_PORT \ - --app-dir /usr/share/fastapi-dls/app \ - --ssl-keyfile /etc/fastapi-dls/webserver.key \ - --ssl-certfile /opt/fastapi-dls/webserver.crt \ +ExecStart=uvicorn main:app \\ + --env-file /etc/fastapi-dls/env \\ + --host \$DLS_URL --port \$DLS_PORT \\ + --app-dir /usr/share/fastapi-dls/app \\ + --ssl-keyfile /etc/fastapi-dls/webserver.key \\ + --ssl-certfile /opt/fastapi-dls/webserver.crt \\ --proxy-headers Restart=always KillSignal=SIGQUIT -Type=notify +Type=simple StandardError=syslog NotifyAccess=all From c820dac4ec78dfbb330acf42ae92a2f53de03bc6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:49:18 +0100 Subject: [PATCH 067/102] README.md - improvements & fixed manual install steps --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 679e026..133408e 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ There are some more internal api endpoints for handling authentication and lease Docker-Images are available here: - [Docker-Hub](https://hub.docker.com/repository/docker/collinwebdesigns/fastapi-dls): `collinwebdesigns/fastapi-dls:latest` -- GitLab-Registry: `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls/main:latest` +- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/container_registry): `registry.git.collinwebdesigns.de/oscar.krause/fastapi-dls/main:latest` **Run this on the Docker-Host** @@ -98,7 +98,7 @@ volumes: dls-db: ``` -## Debian +## Debian/Ubuntu (using `git clone`) Tested on `Debian 11 (bullseye)`, Ubuntu may also work. @@ -148,7 +148,7 @@ su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fast **Create config file** ```shell -cat < /etc/fastapi-dls.env +cat < /etc/fastapi-dls/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 @@ -160,7 +160,7 @@ EOF **Create service** ```shell -cat </etc/systemd/system/fastapi-dls.service +cat < /etc/systemd/system/fastapi-dls.service [Unit] Description=Service for fastapi-dls After=network.target @@ -204,7 +204,7 @@ with `systemctl start fastapi-dls.service` (and enable autostart with `systemctl | `DATABASE` | `sqlite:///db.sqlite` | See [official dataset docs](https://dataset.readthedocs.io/en/latest/quickstart.html) | | `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) | -# Installation (Client) +# Setup (Client) **The token file has to be copied! It's not enough to C&P file contents, because there can be special characters.** From 46620c5e2ad72abdf1ebf78647feee0d5c5c4cb9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:50:04 +0100 Subject: [PATCH 068/102] typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 133408e..d5da347 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ volumes: dls-db: ``` -## Debian/Ubuntu (using `git clone`) +## Debian/Ubuntu (manual method using `git clone`) Tested on `Debian 11 (bullseye)`, Ubuntu may also work. From 8c1c51897ff1de9c75a71b21f8831ae86085a578 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:53:31 +0100 Subject: [PATCH 069/102] README.md - added install instructions --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d5da347..3845c0d 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,27 @@ EOF ``` Now you have to run `systemctl daemon-reload`. After that you can start service -with `systemctl start fastapi-dls.service` (and enable autostart with `systemctl enable fastapi-dls.service`). +with `systemctl start fastapi-dls.service`. + +## Debian/Ubuntu (using `dpkg`) + +Packages are available here: + +- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/packages/63) + +Successful tested with: +- Debian 12 (Bookworm) +- Ubuntu 22.10 (Kinetic Kudu) + +**Run this on your server instance** + +```shell +apt-get update +FILENAME=/opt/fastapi-dls.deb +wget -O $FILENAME https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/package_files/148/download +dpkg -i $FILENAME +apt-get install -f --fix-missing +``` # Configuration From 3b75e8dbeb4ac8c561ab821c2ea79fb2d86d2304 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:54:25 +0100 Subject: [PATCH 070/102] fixes --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d5da347..cd12308 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,13 @@ User=www-data Group=www-data AmbientCapabilities=CAP_NET_BIND_SERVICE WorkingDirectory=/opt/fastapi-dls/app -EnvironmentFile=/etc/fastapi-dls.env -ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \ - --env-file /etc/fastapi-dls.env \ - --host \$DLS_URL --port \$DLS_PORT \ - --app-dir /opt/fastapi-dls/app \ - --ssl-keyfile /opt/fastapi-dls/app/cert/webserver.key \ - --ssl-certfile /opt/fastapi-dls/app/cert/webserver.crt \ +EnvironmentFile=/etc/fastapi-dls/env +ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \\ + --env-file /etc/fastapi-dls/env \\ + --host \$DLS_URL --port \$DLS_PORT \\ + --app-dir /opt/fastapi-dls/app \\ + --ssl-keyfile /opt/fastapi-dls/app/cert/webserver.key \\ + --ssl-certfile /opt/fastapi-dls/app/cert/webserver.crt \\ --proxy-headers Restart=always KillSignal=SIGQUIT From 2af4b456b6bfa1395a6713f0c3ad4bde2f1a4600 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:56:31 +0100 Subject: [PATCH 071/102] fixes --- DEBIAN/postinst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index ea81cfa..83e6ccd 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -10,8 +10,8 @@ After=network.target User=www-data Group=www-data AmbientCapabilities=CAP_NET_BIND_SERVICE -EnvironmentFile=/etc/fastapi-dls/env WorkingDirectory=/usr/share/fastapi-dls +EnvironmentFile=/etc/fastapi-dls/env ExecStart=uvicorn main:app \\ --env-file /etc/fastapi-dls/env \\ --host \$DLS_URL --port \$DLS_PORT \\ @@ -36,8 +36,8 @@ echo "> Create config directory ..." mkdir -p $CONFIG_DIR echo "> Writing default config parameters ..." -touch $CONFIG_DIR/fastapi-dls.env -cat < $CONFIG_DIR/fastapi-dls.env +touch $CONFIG_DIR/fastapi-dls/env +cat < $CONFIG_DIR/fastapi-dls/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 From 84f7e99c786ad03846383c989027a67f369eabbb Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:58:26 +0100 Subject: [PATCH 072/102] README.md - adde toc --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cd12308..4b4af72 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. +[[_TOC_]] + ## ToDo#'s - provide `.deb` package (WIP) From 65937b153e07fc3fc8826d3f556f4cff8c85616f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 06:58:50 +0100 Subject: [PATCH 073/102] typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b4af72..7818c7f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Only the clients need a connection to this service on configured port. [[_TOC_]] -## ToDo#'s +## ToDo's - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) From 63670f52e89b8573620e25418ddf62bc8ab3674d Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:03:41 +0100 Subject: [PATCH 074/102] postinst fixes --- DEBIAN/postinst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 83e6ccd..d1c43e4 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -1,7 +1,12 @@ #!/bin/bash +CONFIG_DIR=/etc/fastapi-dls + +echo "> Create config directory ..." +mkdir -p $CONFIG_DIR + echo "> Install service ..." -cat < /etc/systemd/system/fastapi-dls.service +cat </etc/systemd/system/fastapi-dls.service [Unit] Description=Service for fastapi-dls After=network.target @@ -11,7 +16,7 @@ User=www-data Group=www-data AmbientCapabilities=CAP_NET_BIND_SERVICE WorkingDirectory=/usr/share/fastapi-dls -EnvironmentFile=/etc/fastapi-dls/env +EnvironmentFile=$CONFIG_DIR/env ExecStart=uvicorn main:app \\ --env-file /etc/fastapi-dls/env \\ --host \$DLS_URL --port \$DLS_PORT \\ @@ -30,14 +35,9 @@ WantedBy=multi-user.target EOF -CONFIG_DIR=/etc/fastapi-dls - -echo "> Create config directory ..." -mkdir -p $CONFIG_DIR - echo "> Writing default config parameters ..." -touch $CONFIG_DIR/fastapi-dls/env -cat < $CONFIG_DIR/fastapi-dls/env +touch $CONFIG_DIR/env +cat <$CONFIG_DIR/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 From 9744a8f0e853998bff01c12709f21586f245ae2a Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:04:10 +0100 Subject: [PATCH 075/102] code styling --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7818c7f..fc785e4 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ su - www-data -c "/opt/fastapi-dls/venv/bin/uvicorn main:app --app-dir=/opt/fast **Create config file** ```shell -cat < /etc/fastapi-dls/env +cat </etc/fastapi-dls/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 @@ -162,7 +162,7 @@ EOF **Create service** ```shell -cat < /etc/systemd/system/fastapi-dls.service +cat </etc/systemd/system/fastapi-dls.service [Unit] Description=Service for fastapi-dls After=network.target From a08261f7cd8ecb428b465cf5f500bef29f5d0322 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:14:24 +0100 Subject: [PATCH 076/102] postinst - fixed paths and permissions --- DEBIAN/postinst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index d1c43e4..14700be 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -20,9 +20,9 @@ EnvironmentFile=$CONFIG_DIR/env ExecStart=uvicorn main:app \\ --env-file /etc/fastapi-dls/env \\ --host \$DLS_URL --port \$DLS_PORT \\ - --app-dir /usr/share/fastapi-dls/app \\ + --app-dir /usr/share/fastapi-dls \\ --ssl-keyfile /etc/fastapi-dls/webserver.key \\ - --ssl-certfile /opt/fastapi-dls/webserver.crt \\ + --ssl-certfile /etc/fastapi-dls/webserver.crt \\ --proxy-headers Restart=always KillSignal=SIGQUIT @@ -74,6 +74,8 @@ if [[ -f $CONFIG_DIR/webserver.key ]]; then fi fi +chown -R www-data:www-data $CONFIG_DIR/* + cat < Date: Wed, 28 Dec 2022 07:16:34 +0100 Subject: [PATCH 077/102] postrm - remove service --- DEBIAN/postrm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DEBIAN/postrm b/DEBIAN/postrm index a3d8b25..f786c67 100755 --- a/DEBIAN/postrm +++ b/DEBIAN/postrm @@ -5,4 +5,9 @@ if [[ -d /etc/fastapi-dls ]]; then rm -r /etc/fastapi-dls fi +if [[ -f /etc/systemd/system/fastapi-dls.service ]]; then + echo "> Removing service file." + rm /etc/systemd/system/fastapi-dls.service +fi + # todo From 180cdcb43de8a62dee8c9bd444150a6b9a1dd7f7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:29:38 +0100 Subject: [PATCH 078/102] added some variables --- README.md | 20 ++++++++++++-------- app/main.py | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fc785e4..9247071 100644 --- a/README.md +++ b/README.md @@ -197,14 +197,18 @@ with `systemctl start fastapi-dls.service` (and enable autostart with `systemctl # Configuration -| Variable | Default | Usage | -|---------------------|-----------------------|---------------------------------------------------------------------------------------| -| `DEBUG` | `false` | Toggles `fastapi` debug mode | -| `DLS_URL` | `localhost` | Used in client-token to tell guest driver where dls instance is reachable | -| `DLS_PORT` | `443` | Used in client-token to tell guest driver where dls instance is reachable | -| `LEASE_EXPIRE_DAYS` | `90` | Lease time in days | -| `DATABASE` | `sqlite:///db.sqlite` | See [official dataset docs](https://dataset.readthedocs.io/en/latest/quickstart.html) | -| `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) | +| Variable | Default | Usage | +|---------------------|----------------------------------------|---------------------------------------------------------------------------------------| +| `DEBUG` | `false` | Toggles `fastapi` debug mode | +| `DLS_URL` | `localhost` | Used in client-token to tell guest driver where dls instance is reachable | +| `DLS_PORT` | `443` | Used in client-token to tell guest driver where dls instance is reachable | +| `LEASE_EXPIRE_DAYS` | `90` | Lease time in days | +| `DATABASE` | `sqlite:///db.sqlite` | See [official dataset docs](https://dataset.readthedocs.io/en/latest/quickstart.html) | +| `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) | +| `SITE_KEY_XID` | `00000000-0000-0000-0000-000000000000` | Site identification uuid | +| `INSTANCE_REF` | `00000000-0000-0000-0000-000000000000` | Instance identification uuid | +| `INSTANCE_KEY_RSA` | `/cert/instance.private.pem` | Site-wide private RSA key for singing JWTs | +| `INSTANCE_KEY_PUB` | `/cert/instance.public.pem` | Site-wide public key | # Setup (Client) diff --git a/app/main.py b/app/main.py index 93a2b0b..1ac62e6 100644 --- a/app/main.py +++ b/app/main.py @@ -54,8 +54,8 @@ DLS_URL = str(getenv('DLS_URL', 'localhost')) DLS_PORT = int(getenv('DLS_PORT', '443')) SITE_KEY_XID = str(getenv('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) INSTANCE_REF = str(getenv('INSTANCE_REF', '00000000-0000-0000-0000-000000000000')) -INSTANCE_KEY_RSA = load_key(join(dirname(__file__), 'cert/instance.private.pem')) -INSTANCE_KEY_PUB = load_key(join(dirname(__file__), 'cert/instance.public.pem')) +INSTANCE_KEY_RSA = load_key(str(getenv('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) +INSTANCE_KEY_PUB = load_key(str(getenv('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) CORS_ORIGINS = getenv('CORS_ORIGINS').split(',') if (getenv('CORS_ORIGINS')) else f'https://{DLS_URL}' # todo: prevent static https From a95126f51da4e9d457da75ba6ae7369044e35ea9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:29:42 +0100 Subject: [PATCH 079/102] typos --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9247071..10482b1 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,8 @@ Currently, there are no known issues. ### Required cipher on Windows Guests (e.g. managed by domain controller with GPO) -It is required to enable `SHA1` (`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521`) in [windows cipher suite](https://learn.microsoft.com/en-us/windows-server/security/tls/manage-tls). +It is required to enable `SHA1` (`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521`) +in [windows cipher suite](https://learn.microsoft.com/en-us/windows-server/security/tls/manage-tls). ### Multiple Display Container LS Instances From cca24f0ad56f3bc2e11d54ca67add64135cde6ec Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:31:23 +0100 Subject: [PATCH 080/102] fixed instance keypair path --- DEBIAN/postinst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 14700be..e4e8873 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -42,6 +42,8 @@ DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 DATABASE=sqlite:////usr/share/fastapi-dls/db.sqlite +INSTANCE_KEY_RSA=$CONFIG_DIR/instance.private.pem +INSTANCE_KEY_PUB=$CONFIG_DIR/instance.public.pem EOF From 6b3f536681c86296526786fc80aa4737e1d42b20 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 07:40:44 +0100 Subject: [PATCH 081/102] fixes - fixed app dir - fixed missing readme and version file - keep config on update/remove --- .gitlab-ci.yml | 5 +++-- DEBIAN/postinst | 8 +++++--- DEBIAN/postrm | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e60da24..0c34ce4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,9 +12,10 @@ build:debian: - mkdir build # copy install instructions - cp -r DEBIAN build/ - # copy app + # copy app into "/usr/share/fastapi-dls" as "/usr/share/fastapi-dls/app" & copy README.md and version.env - mkdir -p build/usr/share/ - - cp -r app build/usr/share/fastapi-dls + - cp -r app build/usr/share/fastapi-dls/ + - cp README.md version.env build/usr/share/fastapi-dls # create conf file - mkdir -p build/etc/fastapi-dls - touch build/etc/fastapi-dls/env diff --git a/DEBIAN/postinst b/DEBIAN/postinst index e4e8873..77bf587 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -1,5 +1,6 @@ #!/bin/bash +WORKING_DIR=/usr/share/fastapi-dls CONFIG_DIR=/etc/fastapi-dls echo "> Create config directory ..." @@ -15,12 +16,12 @@ After=network.target User=www-data Group=www-data AmbientCapabilities=CAP_NET_BIND_SERVICE -WorkingDirectory=/usr/share/fastapi-dls +WorkingDirectory=$WORKING_DIR EnvironmentFile=$CONFIG_DIR/env ExecStart=uvicorn main:app \\ --env-file /etc/fastapi-dls/env \\ --host \$DLS_URL --port \$DLS_PORT \\ - --app-dir /usr/share/fastapi-dls \\ + --app-dir $WORKING_DIR/app \\ --ssl-keyfile /etc/fastapi-dls/webserver.key \\ --ssl-certfile /etc/fastapi-dls/webserver.crt \\ --proxy-headers @@ -41,7 +42,7 @@ cat <$CONFIG_DIR/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 -DATABASE=sqlite:////usr/share/fastapi-dls/db.sqlite +DATABASE=sqlite:///$CONFIG_DIR/db.sqlite INSTANCE_KEY_RSA=$CONFIG_DIR/instance.private.pem INSTANCE_KEY_PUB=$CONFIG_DIR/instance.public.pem @@ -77,6 +78,7 @@ if [[ -f $CONFIG_DIR/webserver.key ]]; then fi chown -R www-data:www-data $CONFIG_DIR/* +chown -R www-data:www-data $WORKING_DIR cat < Removing config directory." - rm -r /etc/fastapi-dls + echo "> Config directory not empty, please remove manually." + # rm -r /etc/fastapi-dls fi if [[ -f /etc/systemd/system/fastapi-dls.service ]]; then From cf21bec3b0f26e34e46ac9c165b41f9830aae20d Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:05:35 +0100 Subject: [PATCH 082/102] postrm fixed removing app dir --- DEBIAN/postrm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DEBIAN/postrm b/DEBIAN/postrm index 15aba78..07cb178 100755 --- a/DEBIAN/postrm +++ b/DEBIAN/postrm @@ -5,6 +5,11 @@ if [[ -d /etc/fastapi-dls ]]; then # rm -r /etc/fastapi-dls fi +if [[ -d /usr/share/fastapi-dls/ ]]; then + echo "> Removing app dir." + rm -r /usr/share/fastapi-dls +fi + if [[ -f /etc/systemd/system/fastapi-dls.service ]]; then echo "> Removing service file." rm /etc/systemd/system/fastapi-dls.service From 45af6c11c0872c745a056639ca9eec648bab2ad8 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:21:04 +0100 Subject: [PATCH 083/102] fixed missing systemctl daemon-reload --- DEBIAN/postinst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 77bf587..080dc6c 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -36,6 +36,8 @@ WantedBy=multi-user.target EOF +systemctl daemon-reload + echo "> Writing default config parameters ..." touch $CONFIG_DIR/env cat <$CONFIG_DIR/env From 6844604a0b59bdb473158b34a394f14cfdeb6858 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:25:31 +0100 Subject: [PATCH 084/102] fixed deb package paths --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c34ce4..fc06da2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,8 @@ build:debian: # copy install instructions - cp -r DEBIAN build/ # copy app into "/usr/share/fastapi-dls" as "/usr/share/fastapi-dls/app" & copy README.md and version.env - - mkdir -p build/usr/share/ - - cp -r app build/usr/share/fastapi-dls/ + - mkdir -p build/usr/share/fastapi-dls + - cp -r app build/usr/share/fastapi-dls - cp README.md version.env build/usr/share/fastapi-dls # create conf file - mkdir -p build/etc/fastapi-dls From da21ef3cdc5d96eeb5e4be2f12e89ceeeba9bdae Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:35:59 +0100 Subject: [PATCH 085/102] fixed some permissions --- DEBIAN/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 080dc6c..6376347 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -79,7 +79,7 @@ if [[ -f $CONFIG_DIR/webserver.key ]]; then fi fi -chown -R www-data:www-data $CONFIG_DIR/* +chown -R www-data:www-data $CONFIG_DIR chown -R www-data:www-data $WORKING_DIR cat < Date: Wed, 28 Dec 2022 08:37:34 +0100 Subject: [PATCH 086/102] README.md - added Let's Encrypt section --- README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 10482b1..ed407a8 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Only the clients need a connection to this service on configured port. - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) - migrate from `fastapi` to `flask` -- Support http mode for using external https proxy +- Support http mode for using external https proxy (disable uvicorn ssl for using behind proxy) ## Endpoints @@ -193,7 +193,23 @@ EOF ``` Now you have to run `systemctl daemon-reload`. After that you can start service -with `systemctl start fastapi-dls.service` (and enable autostart with `systemctl enable fastapi-dls.service`). +with `systemctl start fastapi-dls.service`. + +## Let's Encrypt Certificate + +If you're using installation via docker, you can use `traefik`. Please refer to their documentation. + +Note that port 80 must be accessible, and you have to install `socat` if you're using `standalone` mode. + +```shell +acme.sh --issue -d example.com \ + --cert-file /etc/fastapi-dls/webserver.donotuse.crt \ + --key-file /etc/fastapi-dls/webserver.key \ + --fullchain-file /etc/fastapi-dls/webserver.crt \ + --reloadcmd "systemctl restart fastapi-dls.service" +``` + +After first success you have to replace `--issue` with `--renew`. # Configuration From b745367baac8dba3b7c817fa6a857d5cbc6ae47c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:48:36 +0100 Subject: [PATCH 087/102] postrm fixed --- DEBIAN/postrm | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/DEBIAN/postrm b/DEBIAN/postrm index 07cb178..b99d0fa 100755 --- a/DEBIAN/postrm +++ b/DEBIAN/postrm @@ -1,15 +1,5 @@ #!/bin/bash -if [[ -d /etc/fastapi-dls ]]; then - echo "> Config directory not empty, please remove manually." - # rm -r /etc/fastapi-dls -fi - -if [[ -d /usr/share/fastapi-dls/ ]]; then - echo "> Removing app dir." - rm -r /usr/share/fastapi-dls -fi - if [[ -f /etc/systemd/system/fastapi-dls.service ]]; then echo "> Removing service file." rm /etc/systemd/system/fastapi-dls.service From 4e5559bb85248a23a95d856b6310f34295994d7c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:51:55 +0100 Subject: [PATCH 088/102] fixed service Standard output type syslog is obsolete, automatically updating to journal. Please update your unit file, and consider removing the setting altogether. --- DEBIAN/postinst | 1 - 1 file changed, 1 deletion(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 6376347..307ee2f 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -28,7 +28,6 @@ ExecStart=uvicorn main:app \\ Restart=always KillSignal=SIGQUIT Type=simple -StandardError=syslog NotifyAccess=all [Install] From e9dc5a765a48d23e3e01438f5f69d5fe20d2bbd8 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:52:13 +0100 Subject: [PATCH 089/102] fixed service Standard output type syslog is obsolete, automatically updating to journal. Please update your unit file, and consider removing the setting altogether. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ed407a8..120feed 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,6 @@ ExecStart=/opt/fastapi-dls/venv/bin/uvicorn main:app \\ Restart=always KillSignal=SIGQUIT Type=simple -StandardError=syslog NotifyAccess=all [Install] From 437b62376fc0984c46b731ca72692583fa4040ee Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:56:11 +0100 Subject: [PATCH 090/102] fixed missing debian dependency --- DEBIAN/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEBIAN/control b/DEBIAN/control index aa81b51..1eeb2b2 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -2,7 +2,7 @@ Package: fastapi-dls Version: 0.6.0 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de -Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, uvicorn, openssl +Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl Recommends: curl Installed-Size: 10240 Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls From 2340931a6066340b47dd6f76613c86e3b496f3a3 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 08:57:35 +0100 Subject: [PATCH 091/102] fixes --- DEBIAN/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index 307ee2f..a0e7334 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -16,7 +16,7 @@ After=network.target User=www-data Group=www-data AmbientCapabilities=CAP_NET_BIND_SERVICE -WorkingDirectory=$WORKING_DIR +WorkingDirectory=$WORKING_DIR/app EnvironmentFile=$CONFIG_DIR/env ExecStart=uvicorn main:app \\ --env-file /etc/fastapi-dls/env \\ From b22613c3379408336efe7af6108b1ad098f2a46c Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:04:35 +0100 Subject: [PATCH 092/102] postinst improvements --- DEBIAN/postinst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index a0e7334..2311b35 100644 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -37,9 +37,10 @@ EOF systemctl daemon-reload -echo "> Writing default config parameters ..." -touch $CONFIG_DIR/env -cat <$CONFIG_DIR/env +if [[ ! -f $CONFIG_DIR/env ]]; then + echo "> Writing initial config ..." + touch $CONFIG_DIR/env + cat <$CONFIG_DIR/env DLS_URL=127.0.0.1 DLS_PORT=443 LEASE_EXPIRE_DAYS=90 @@ -48,6 +49,7 @@ INSTANCE_KEY_RSA=$CONFIG_DIR/instance.private.pem INSTANCE_KEY_PUB=$CONFIG_DIR/instance.public.pem EOF +fi echo "> Create dls-instance keypair ..." openssl genrsa -out $CONFIG_DIR/instance.private.pem 2048 @@ -72,7 +74,8 @@ if [[ -f $CONFIG_DIR/webserver.key ]]; then if [ -x "$(command -v curl)" ]; then echo "> Testing API ..." - curl --insecure -X GET https://127.0.0.1/status + source $CONFIG_DIR/env + curl --insecure -X GET https://$DLS_URL:$DLS_PORT/status else echo "> Testing API failed, curl not available. Please test manually!" fi From 3dc9c8bcb19a7166edbedccc27c412cec723d878 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:10:57 +0100 Subject: [PATCH 093/102] README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1dfabe7..358ceae 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ EOF ``` Now you have to run `systemctl daemon-reload`. After that you can start service -with `systemctl start fastapi-dls.service`. +with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`. ## Debian/Ubuntu (using `dpkg`) @@ -214,7 +214,8 @@ wget -O $FILENAME https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/pac dpkg -i $FILENAME apt-get install -f --fix-missing ``` -with `systemctl start fastapi-dls.service`. + +Start with `systemctl start fastapi-dls.service` and enable autostart with `systemctl enable fastapi-dls.service`. ## Let's Encrypt Certificate From e1f2e942a6bc44a279f5279fb9615f76ab778765 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:23:17 +0100 Subject: [PATCH 094/102] code styling --- app/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/main.py b/app/main.py index e3778ef..4594ba5 100644 --- a/app/main.py +++ b/app/main.py @@ -27,7 +27,6 @@ try: except ModuleNotFoundError: from Cryptodome.PublicKey import RSA from Cryptodome.PublicKey.RSA import RsaKey - from orm import Origin, Lease, init as db_init logger = logging.getLogger() @@ -35,6 +34,7 @@ load_dotenv('../version.env') VERSION, COMMIT, DEBUG = getenv('VERSION', 'unknown'), getenv('COMMIT', 'unknown'), bool(getenv('DEBUG', False)) + def load_file(filename) -> bytes: with open(filename, 'rb') as file: content = file.read() @@ -56,15 +56,14 @@ __details = dict( app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) db_init(db) -TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 -LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) - DLS_URL = str(getenv('DLS_URL', 'localhost')) DLS_PORT = int(getenv('DLS_PORT', '443')) SITE_KEY_XID = str(getenv('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) INSTANCE_REF = str(getenv('INSTANCE_REF', '00000000-0000-0000-0000-000000000000')) INSTANCE_KEY_RSA = load_key(str(getenv('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) INSTANCE_KEY_PUB = load_key(str(getenv('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) +TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 +LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) CORS_ORIGINS = getenv('CORS_ORIGINS').split(',') if (getenv('CORS_ORIGINS')) else f'https://{DLS_URL}' # todo: prevent static https From 89bf744054a285e96a841b5765d482cb43c59997 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:24:02 +0100 Subject: [PATCH 095/102] removed some todos --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 4eee1e0..6a43feb 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,6 @@ Only the clients need a connection to this service on configured port. ## ToDo's -- provide `.deb` package (WIP) -- migrate from `dataset` to `sqlalchemy` (WIP) - migrate from `fastapi` to `flask` - Support http mode for using external https proxy (disable uvicorn ssl for using behind proxy) From 3d073dbd7d3c47ece92f897cb6b52fd6a9418a0f Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:24:41 +0100 Subject: [PATCH 096/102] bump version to 1.0.0 --- DEBIAN/control | 2 +- version.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DEBIAN/control b/DEBIAN/control index 1eeb2b2..01db5e9 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,5 +1,5 @@ Package: fastapi-dls -Version: 0.6.0 +Version: 1.0.0 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl diff --git a/version.env b/version.env index f7fc9a2..624bade 100644 --- a/version.env +++ b/version.env @@ -1 +1 @@ -VERSION=0.6 +VERSION=1.0.0 From 0e24d26089df63cb1f71d0a30486041993b815bd Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:47:31 +0100 Subject: [PATCH 097/102] README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6a43feb..7c43249 100644 --- a/README.md +++ b/README.md @@ -196,18 +196,22 @@ with `systemctl start fastapi-dls.service` and enable autostart with `systemctl Packages are available here: -- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/packages/63) +- [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/packages) Successful tested with: + - Debian 12 (Bookworm) - Ubuntu 22.10 (Kinetic Kudu) **Run this on your server instance** +First go to [GitLab-Registry](https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/packages) and select your +version. Then you have to copy the download link of the `fastapi-dls_X.Y.Z_amd64.deb` asset. + ```shell apt-get update FILENAME=/opt/fastapi-dls.deb -wget -O $FILENAME https://git.collinwebdesigns.de/oscar.krause/fastapi-dls/-/package_files/148/download +wget -O $FILENAME dpkg -i $FILENAME apt-get install -f --fix-missing ``` From e88b1afcf78d31d83cd495746cc1bdef51d9da25 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 09:57:55 +0100 Subject: [PATCH 098/102] fixes --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 4594ba5..6a7838a 100644 --- a/app/main.py +++ b/app/main.py @@ -259,7 +259,7 @@ async def auth_v1_code(request: Request): @app.post('/auth/v1/token') async def auth_v1_token(request: Request): j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() - payload = jwt.decode(token=j['auth_code'], key=jwt_decode_key) + payload = jwt.decode(token=j['auth_code'], key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) origin_ref = payload['origin_ref'] logging.info(f'> [ auth ]: {origin_ref}: {j}') From 670e05f69326b9fe244852a6821630b2bde75b91 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 10:00:34 +0100 Subject: [PATCH 099/102] .gitlab-ci.yml --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc06da2..9c02650 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,8 +113,8 @@ deploy:debian: # doc: https://git.collinwebdesigns.de/help/user/packages/debian_repository/index.md#install-a-package image: debian:bookworm-slim stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == "debian" # $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +# rules: +# - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH needs: - job: build:debian artifacts: true From dada9cc4cdce550f6840c7900245f9325ecebbc4 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 11:05:41 +0100 Subject: [PATCH 100/102] fixes --- app/main.py | 2 +- app/orm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 6a7838a..2c45354 100644 --- a/app/main.py +++ b/app/main.py @@ -53,7 +53,7 @@ __details = dict( version=VERSION, ) -app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) +app, db = FastAPI(**__details), create_engine(str(getenv('DATABASE', 'sqlite:///db.sqlite'))) db_init(db) DLS_URL = str(getenv('DLS_URL', 'localhost')) diff --git a/app/orm.py b/app/orm.py index 26b869a..697c720 100644 --- a/app/orm.py +++ b/app/orm.py @@ -2,7 +2,7 @@ import datetime from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete, inspect from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.future import Engine +from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker Base = declarative_base() From a951433ca0e89c48267acc55d8c38e4ae88f71e0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 11:33:06 +0100 Subject: [PATCH 101/102] fixes --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 2c45354..b21c8c1 100644 --- a/app/main.py +++ b/app/main.py @@ -68,7 +68,7 @@ LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) CORS_ORIGINS = getenv('CORS_ORIGINS').split(',') if (getenv('CORS_ORIGINS')) else f'https://{DLS_URL}' # todo: prevent static https jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) -jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS512) +jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) app.debug = DEBUG app.add_middleware( @@ -259,7 +259,7 @@ async def auth_v1_code(request: Request): @app.post('/auth/v1/token') async def auth_v1_token(request: Request): j, cur_time = json.loads((await request.body()).decode('utf-8')), datetime.utcnow() - payload = jwt.decode(token=j['auth_code'], key=jwt_decode_key, algorithms=ALGORITHMS.RS256, options={'verify_aud': False}) + payload = jwt.decode(token=j['auth_code'], key=jwt_decode_key) origin_ref = payload['origin_ref'] logging.info(f'> [ auth ]: {origin_ref}: {j}') From c83130f13841548f3eea2fe681feee147572ed89 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Wed, 28 Dec 2022 11:33:26 +0100 Subject: [PATCH 102/102] README.md - added known issue --- README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c43249..095f975 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,29 @@ Logs are available in `C:\Users\Public\Documents\Nvidia\LoggingLog.NVDisplay.Con ## Linux -Currently, there are no known issues. +### `uvicorn.error:Invalid HTTP request received.` + +This message can be ignored. + +- Ref. https://github.com/encode/uvicorn/issues/441 + +``` +WARNING:uvicorn.error:Invalid HTTP request received. +Traceback (most recent call last): + File "/usr/lib/python3/dist-packages/uvicorn/protocols/http/h11_impl.py", line 129, in handle_events + event = self.conn.next_event() + File "/usr/lib/python3/dist-packages/h11/_connection.py", line 485, in next_event + exc._reraise_as_remote_protocol_error() + File "/usr/lib/python3/dist-packages/h11/_util.py", line 77, in _reraise_as_remote_protocol_error + raise self + File "/usr/lib/python3/dist-packages/h11/_connection.py", line 467, in next_event + event = self._extract_next_receive_event() + File "/usr/lib/python3/dist-packages/h11/_connection.py", line 409, in _extract_next_receive_event + event = self._reader(self._receive_buffer) + File "/usr/lib/python3/dist-packages/h11/_readers.py", line 84, in maybe_read_from_IDLE_client + raise LocalProtocolError("no request line received") +h11._util.RemoteProtocolError: no request line received +``` ## Windows