From 394180652ea8c5e7215307fd78a1525d6ca19319 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Thu, 22 Dec 2022 12:57:06 +0100 Subject: [PATCH 01/15] migrated from dataset to sqlalchemy --- app/main.py | 68 ++++++++++------ app/orm.py | 198 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 244 insertions(+), 24 deletions(-) create mode 100644 app/orm.py diff --git a/app/main.py b/app/main.py index f35d3fc..279a12e 100644 --- a/app/main.py +++ b/app/main.py @@ -17,16 +17,18 @@ from jose import jws, jwk, jwt from jose.constants import ALGORITHMS from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse, HTMLResponse -import dataset +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey +from orm import Origin, Lease, Auth + logger = logging.getLogger() load_dotenv('../version.env') VERSION, COMMIT, DEBUG = getenv('VERSION', 'unknown'), getenv('COMMIT', 'unknown'), bool(getenv('DEBUG', False)) - def load_file(filename) -> bytes: with open(filename, 'rb') as file: content = file.read() @@ -45,7 +47,7 @@ __details = dict( version=VERSION, ) -app, db = FastAPI(**__details), dataset.connect(str(getenv('DATABASE', 'sqlite:///db.sqlite'))) +app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) @@ -93,13 +95,17 @@ async def status(request: Request): @app.get('/-/origins') async def _origins(request: Request): - response = list(map(lambda x: jsonable_encoder(x), db['origin'].all())) + session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + response = list(map(lambda x: jsonable_encoder(x), session.query(Origin).all())) + session.close() return JSONResponse(response) @app.get('/-/leases') async def _leases(request: Request): - response = list(map(lambda x: jsonable_encoder(x), db['lease'].all())) + session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + response = list(map(lambda x: jsonable_encoder(x), session.query(Lease).all())) + session.close() return JSONResponse(response) @@ -160,14 +166,14 @@ async def auth_v1_origin(request: Request): origin_ref = j['candidate_origin_ref'] logging.info(f'> [ origin ]: {origin_ref}: {j}') - data = dict( + data = Origin( origin_ref=origin_ref, hostname=j['environment']['hostname'], guest_driver_version=j['environment']['guest_driver_version'], os_platform=j['environment']['os_platform'], os_version=j['environment']['os_version'], ) - db['origin'].upsert(data, ['origin_ref']) + Origin.create_or_update(db, data) cur_time = datetime.utcnow() response = { @@ -208,8 +214,9 @@ async def auth_v1_code(request: Request): auth_code = jws.sign(payload, key=jwt_encode_key, headers={'kid': payload.get('kid')}, algorithm=ALGORITHMS.RS256) - db['auth'].delete(origin_ref=origin_ref, expires={'<=': cur_time - delta}) - db['auth'].insert(dict(origin_ref=origin_ref, code_challenge=j['code_challenge'], expires=expires)) + Auth.cleanup(db, origin_ref, cur_time - delta) + data = Auth(origin_ref=origin_ref, code_challenge=j['code_challenge'], expires=expires) + Auth.create(db, data) response = { "auth_code": auth_code, @@ -230,7 +237,10 @@ async def auth_v1_token(request: Request): code_challenge = payload['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ auth ]: {origin_ref} ({code_challenge}): {j}') # validate the code challenge @@ -270,8 +280,10 @@ async def leasing_v1_lessor(request: Request): code_challenge = token['origin_ref'] scope_ref_list = j['scope_ref_list'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ create ]: {origin_ref} ({code_challenge}): create leases for scope_ref_list {scope_ref_list}') cur_time = datetime.utcnow() @@ -292,8 +304,8 @@ async def leasing_v1_lessor(request: Request): } }) - data = dict(origin_ref=origin_ref, lease_ref=scope_ref, lease_created=cur_time, lease_expires=expires) - db['lease'].insert_ignore(data, ['origin_ref', 'lease_ref']) # todo: handle update + data = Lease(origin_ref=origin_ref, lease_ref=scope_ref, lease_created=cur_time, lease_expires=expires) + Lease.create_or_update(db, data) response = { "lease_result_list": lease_result_list, @@ -313,8 +325,11 @@ async def leasing_v1_lessor_lease(request: Request): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - active_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref + active_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) logging.info(f'> [ leases ]: {origin_ref} ({code_challenge}): found {len(active_lease_list)} active leases') cur_time = datetime.utcnow() @@ -334,10 +349,14 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref logging.info(f'> [ renew ]: {origin_ref} ({code_challenge}): renew {lease_ref}') - if db['lease'].count(origin_ref=origin_ref, lease_ref=lease_ref) == 0: + entity = Lease.find_by_origin_ref_and_lease_ref(db, origin_ref, lease_ref) + if entity is None: raise HTTPException(status_code=404, detail='requested lease not available') cur_time = datetime.utcnow() @@ -351,8 +370,7 @@ async def leasing_v1_lease_renew(request: Request, lease_ref: str): "sync_timestamp": cur_time.isoformat(), } - data = dict(origin_ref=origin_ref, lease_ref=lease_ref, lease_expires=expires, lease_last_update=cur_time) - db['lease'].update(data, ['origin_ref', 'lease_ref']) + Lease.renew(db, entity, expires, cur_time) return JSONResponse(response) @@ -363,9 +381,13 @@ async def leasing_v1_lessor_lease_remove(request: Request): code_challenge = token['origin_ref'] - origin_ref = db['auth'].find_one(code_challenge=code_challenge)['origin_ref'] - released_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) - deletions = db['lease'].delete(origin_ref=origin_ref) + entity = Auth.find_by_code_challenge(db, code_challenge) + if entity is None: + raise HTTPException(status_code=400, detail='code challenge not found') + origin_ref = entity.origin_ref + + released_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) + deletions = Lease.ceanup(db, origin_ref) logging.info(f'> [ remove ]: {origin_ref} ({code_challenge}): removed {deletions} leases') cur_time = datetime.utcnow() diff --git a/app/orm.py b/app/orm.py new file mode 100644 index 0000000..e253b18 --- /dev/null +++ b/app/orm.py @@ -0,0 +1,198 @@ +import datetime + +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.future import Engine +from sqlalchemy.orm import sessionmaker + +Base = declarative_base() + + +class Origin(Base): + __tablename__ = "origin" + + """ + CREATE TABLE origin ( + id INTEGER NOT NULL, + origin_ref TEXT, + hostname TEXT, + guest_driver_version TEXT, + os_platform TEXT, + os_version TEXT, + PRIMARY KEY (id) + ); + CREATE INDEX ix_origin_0548dd22f20de1bb ON origin (origin_ref); + """ + + """ + 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|ubuntu-grid-server|525.60.13|Ubuntu 20.04|20.04.5 LTS (Focal Fossa) + 2|230b0000-a356-4000-8a2b-0000564c0000|PC-WORKSTATION|527.41|Windows 10 Pro|10.0.19045 + 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|docker-cuda-1|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) + 4|41720000-FA43-4000-9472-0000E8660000|PC-Windows|527.41|Windows 10 Pro|10.0.19045 + 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|docker-cuda-2|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) + """ + + origin_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4 + + hostname = Column(VARCHAR(length=256), nullable=True) + guest_driver_version = Column(VARCHAR(length=10), nullable=True) + os_platform = Column(VARCHAR(length=256), nullable=True) + os_version = Column(VARCHAR(length=256), nullable=True) + + def __repr__(self): + return f'Origin(origin_ref={self.origin_ref}, hostname={self.hostname})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Origin.__table__).compile(engine) + + @staticmethod + def create_or_update(engine: Engine, origin: "Origin"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Origin).filter(Origin.origin_ref == origin.origin_ref).first() + print(entity) + if entity is None: + session.add(origin) + else: + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**origin.values())) + session.flush() + session.close() + + +class Auth(Base): + __tablename__ = "auth" + + """ + CREATE TABLE auth ( + id INTEGER NOT NULL, + origin_ref TEXT, + code_challenge TEXT, + expires DATETIME, + PRIMARY KEY (id) + ); + """ + + """ + 20|B210CF72-FEC7-4440-9499-1156D1ACD13A|p8oeBJPumrosywezCQ6VQvI/J2LZbMRK0s+OfsqzAiI|2022-12-21 05:00:57.467359 + 61|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|9Nnv5FMtV9nF8qYRtCKG5lfF23HGvvNCQvpCh3FUITo|2022-12-22 05:08:40.713022 + 65|230b0000-a356-4000-8a2b-0000564c0000|9PivDr3PYRfcdUgODBR5+gi2ZdAbmPb07yTO05uui4A|2022-12-22 06:22:27.409642 + 66|41720000-FA43-4000-9472-0000E8660000|VnyasehSayRX/2OD3YyP8Xn9nsIBVefZpscnIIj2Rpk|2022-12-22 08:58:04.279664 + 67|41720000-FA43-4000-9472-0000E8660000|uisrxDFKB8KuD+JvtgT1ol5pNm/GKKlhO69u2ntg7z0|2022-12-22 08:59:37.509520 + 68|908B202D-CC43-420F-A2EF-FC092AAE8D38|VtWk7It+k33FxiGjm9rlSgAg1ZigfreFJd/0tt30FgQ|2022-12-22 09:43:56.680163 + """ + + _table_args__ = ( + # this can be db.PrimaryKeyConstraint if you want it to be a primary key + UniqueConstraint('origin_ref', 'code_challenge'), + ) + + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) + code_challenge = Column(VARCHAR(length=43), primary_key=True, nullable=False) + expires = Column(DATETIME(), nullable=False) + + def __repr__(self): + return f'Auth(origin_ref={self.origin_ref}, code_challenge={self.code_challenge}, expires={self.expires})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Auth.__table__).compile(engine) + + @staticmethod + def create(engine: Engine, auth: "Auth"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + session.add(auth) + session.flush() + session.close() + + @staticmethod + def cleanup(engine: Engine, origin_ref: str, older_than: datetime.datetime): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + session.execute(delete(Auth).where(and_(Auth.origin_ref == origin_ref, Auth.expires <= older_than))) + session.close() + + @staticmethod + def find_by_code_challenge(engine: Engine, code_challenge: str) -> "Auth": + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Auth).filter(Auth.code_challenge == code_challenge).first() + session.close() + return entity + + +class Lease(Base): + __tablename__ = "lease" + + """ + CREATE TABLE lease ( + id INTEGER NOT NULL, + origin_ref TEXT, + lease_ref TEXT, + lease_created DATETIME, + lease_expires DATETIME, lease_last_update DATETIME, + PRIMARY KEY (id) + ); + CREATE INDEX ix_lease_11c7d13bfb17f70d ON lease (origin_ref, lease_ref); + """ + + """ + 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|9c4536f9-a216-44c7-a1d3-388a15ee80be|2022-12-20 17:29:07.906668|2022-12-22 04:45:58.138211|2022-12-21 04:45:58.138211 + 2|230b0000-a356-4000-8a2b-0000564c0000|1d95e160-058d-4052-b49f-b85306b4c345|2022-12-20 17:30:25.388389|2022-12-23 06:07:29.913027|2022-12-22 06:07:29.913027 + 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|9e1bca05-e247-4847-9de6-8b9a210b353e|2022-12-20 17:31:40.158003|2022-12-23 09:28:57.379008|2022-12-22 09:28:57.379008 + 4|41720000-FA43-4000-9472-0000E8660000|f2ece7fa-d0c6-4af4-901c-6d3b2c3ecf88|2022-12-20 21:03:33.403711|2022-12-23 08:44:39.998754|2022-12-22 08:44:39.998754 + 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|5455f59b-dd70-45c1-82fa-3fd5fae6c037|2022-12-21 06:05:35.085572|2022-12-23 04:53:41.385178|2022-12-22 04:53:41.385178 + """ + + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) + lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) + lease_created = Column(DATETIME(), nullable=False) + lease_expires = Column(DATETIME(), nullable=False) + lease_updated = Column(DATETIME(), nullable=False) + + def __repr__(self): + return f'Lease(origin_ref={self.origin_ref}, lease_ref={self.lease_ref}, expires={self.lease_expires})' + + @staticmethod + def create_statement(engine: Engine): + from sqlalchemy.schema import CreateTable + return CreateTable(Lease.__table__).compile(engine) + + @staticmethod + def create_or_update(engine: Engine, lease: "Lease"): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Lease).filter(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).first() + if entity is None: + session.add(lease) + else: + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + session.flush() + session.close() + + @staticmethod + def find_by_origin_ref(engine: Engine, origin_ref: str) -> ["Lease"]: + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entities = session.query(Lease).filter(Lease.origin_ref == origin_ref).all() + session.close() + return entities + + @staticmethod + def find_by_origin_ref_and_lease_ref(engine: Engine, origin_ref: str, lease_ref: str) -> "Lease": + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + entity = session.query(Lease).filter(and_(Lease.origin_ref == origin_ref, Lease.lease_ref == lease_ref)).first() + session.close() + return entity + + @staticmethod + def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + lease.lease_expires = lease_expires + lease.lease_updated = lease_updated + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + session.close() + + @staticmethod + def cleanup(engine: Engine, origin_ref: str) -> int: + session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() + deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) + session.close() + return deletions diff --git a/requirements.txt b/requirements.txt index 413b6d1..cceb6a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ uvicorn[standard]==0.20.0 python-jose==3.3.0 pycryptodome==3.16.0 python-dateutil==2.8.2 -dataset==1.5.2 +sqlalchemy==1.4.45 markdown==3.4.1 python-dotenv==0.21.0 From d1db441df4d8719f23a51c2261db2ec0bcbe65f0 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:16:34 +0100 Subject: [PATCH 02/15] removed Auth --- app/main.py | 2 +- app/orm.py | 60 ----------------------------------------------------- 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/app/main.py b/app/main.py index ae3b934..15f0c35 100644 --- a/app/main.py +++ b/app/main.py @@ -22,7 +22,7 @@ from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey -from orm import Origin, Lease, Auth +from orm import Origin, Lease logger = logging.getLogger() load_dotenv('../version.env') diff --git a/app/orm.py b/app/orm.py index e253b18..caae193 100644 --- a/app/orm.py +++ b/app/orm.py @@ -60,66 +60,6 @@ class Origin(Base): session.close() -class Auth(Base): - __tablename__ = "auth" - - """ - CREATE TABLE auth ( - id INTEGER NOT NULL, - origin_ref TEXT, - code_challenge TEXT, - expires DATETIME, - PRIMARY KEY (id) - ); - """ - - """ - 20|B210CF72-FEC7-4440-9499-1156D1ACD13A|p8oeBJPumrosywezCQ6VQvI/J2LZbMRK0s+OfsqzAiI|2022-12-21 05:00:57.467359 - 61|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|9Nnv5FMtV9nF8qYRtCKG5lfF23HGvvNCQvpCh3FUITo|2022-12-22 05:08:40.713022 - 65|230b0000-a356-4000-8a2b-0000564c0000|9PivDr3PYRfcdUgODBR5+gi2ZdAbmPb07yTO05uui4A|2022-12-22 06:22:27.409642 - 66|41720000-FA43-4000-9472-0000E8660000|VnyasehSayRX/2OD3YyP8Xn9nsIBVefZpscnIIj2Rpk|2022-12-22 08:58:04.279664 - 67|41720000-FA43-4000-9472-0000E8660000|uisrxDFKB8KuD+JvtgT1ol5pNm/GKKlhO69u2ntg7z0|2022-12-22 08:59:37.509520 - 68|908B202D-CC43-420F-A2EF-FC092AAE8D38|VtWk7It+k33FxiGjm9rlSgAg1ZigfreFJd/0tt30FgQ|2022-12-22 09:43:56.680163 - """ - - _table_args__ = ( - # this can be db.PrimaryKeyConstraint if you want it to be a primary key - UniqueConstraint('origin_ref', 'code_challenge'), - ) - - origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) - code_challenge = Column(VARCHAR(length=43), primary_key=True, nullable=False) - expires = Column(DATETIME(), nullable=False) - - def __repr__(self): - return f'Auth(origin_ref={self.origin_ref}, code_challenge={self.code_challenge}, expires={self.expires})' - - @staticmethod - def create_statement(engine: Engine): - from sqlalchemy.schema import CreateTable - return CreateTable(Auth.__table__).compile(engine) - - @staticmethod - def create(engine: Engine, auth: "Auth"): - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - session.add(auth) - session.flush() - session.close() - - @staticmethod - def cleanup(engine: Engine, origin_ref: str, older_than: datetime.datetime): - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - session.execute(delete(Auth).where(and_(Auth.origin_ref == origin_ref, Auth.expires <= older_than))) - session.close() - - @staticmethod - def find_by_code_challenge(engine: Engine, code_challenge: str) -> "Auth": - session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - entity = session.query(Auth).filter(Auth.code_challenge == code_challenge).first() - session.close() - return entity - - class Lease(Base): __tablename__ = "lease" From e7102c4de655756102d7b76d84de9aebc973a9dc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:16:58 +0100 Subject: [PATCH 03/15] fixed updates --- app/orm.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/orm.py b/app/orm.py index caae193..0492687 100644 --- a/app/orm.py +++ b/app/orm.py @@ -55,7 +55,13 @@ class Origin(Base): if entity is None: session.add(origin) else: - session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**origin.values())) + values = dict( + hostname=origin.hostname, + guest_driver_version=origin.guest_driver_version, + os_platform=origin.os_platform, + os_version=origin.os_version, + ) + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(values)) session.flush() session.close() @@ -104,7 +110,8 @@ class Lease(Base): if entity is None: session.add(lease) else: - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) session.flush() session.close() @@ -125,9 +132,8 @@ class Lease(Base): @staticmethod def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - lease.lease_expires = lease_expires - lease.lease_updated = lease_updated - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**lease.values())) + values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) session.close() @staticmethod From 43d5736f37cd87806f7c19c92689d7edbb4f6cfc Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 08:22:21 +0100 Subject: [PATCH 04/15] code styling & removed comments --- app/orm.py | 44 ++------------------------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/app/orm.py b/app/orm.py index 0492687..b7b05b7 100644 --- a/app/orm.py +++ b/app/orm.py @@ -11,27 +11,6 @@ Base = declarative_base() class Origin(Base): __tablename__ = "origin" - """ - CREATE TABLE origin ( - id INTEGER NOT NULL, - origin_ref TEXT, - hostname TEXT, - guest_driver_version TEXT, - os_platform TEXT, - os_version TEXT, - PRIMARY KEY (id) - ); - CREATE INDEX ix_origin_0548dd22f20de1bb ON origin (origin_ref); - """ - - """ - 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|ubuntu-grid-server|525.60.13|Ubuntu 20.04|20.04.5 LTS (Focal Fossa) - 2|230b0000-a356-4000-8a2b-0000564c0000|PC-WORKSTATION|527.41|Windows 10 Pro|10.0.19045 - 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|docker-cuda-1|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) - 4|41720000-FA43-4000-9472-0000E8660000|PC-Windows|527.41|Windows 10 Pro|10.0.19045 - 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|docker-cuda-2|525.60.13|Debian GNU/Linux 10 (buster) 10|10 (buster) - """ - origin_ref = Column(CHAR(length=36), primary_key=True, unique=True, index=True) # uuid4 hostname = Column(VARCHAR(length=256), nullable=True) @@ -69,28 +48,9 @@ class Origin(Base): class Lease(Base): __tablename__ = "lease" - """ - CREATE TABLE lease ( - id INTEGER NOT NULL, - origin_ref TEXT, - lease_ref TEXT, - lease_created DATETIME, - lease_expires DATETIME, lease_last_update DATETIME, - PRIMARY KEY (id) - ); - CREATE INDEX ix_lease_11c7d13bfb17f70d ON lease (origin_ref, lease_ref); - """ + origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) # uuid4 + lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) # uuid4 - """ - 1|B210CF72-FEC7-4440-9499-1156D1ACD13A|9c4536f9-a216-44c7-a1d3-388a15ee80be|2022-12-20 17:29:07.906668|2022-12-22 04:45:58.138211|2022-12-21 04:45:58.138211 - 2|230b0000-a356-4000-8a2b-0000564c0000|1d95e160-058d-4052-b49f-b85306b4c345|2022-12-20 17:30:25.388389|2022-12-23 06:07:29.913027|2022-12-22 06:07:29.913027 - 3|908B202D-CC43-420F-A2EF-FC092AAE8D38|9e1bca05-e247-4847-9de6-8b9a210b353e|2022-12-20 17:31:40.158003|2022-12-23 09:28:57.379008|2022-12-22 09:28:57.379008 - 4|41720000-FA43-4000-9472-0000E8660000|f2ece7fa-d0c6-4af4-901c-6d3b2c3ecf88|2022-12-20 21:03:33.403711|2022-12-23 08:44:39.998754|2022-12-22 08:44:39.998754 - 5|723EA079-7B0C-4E25-A8D4-DD3E89F9D177|5455f59b-dd70-45c1-82fa-3fd5fae6c037|2022-12-21 06:05:35.085572|2022-12-23 04:53:41.385178|2022-12-22 04:53:41.385178 - """ - - origin_ref = Column(CHAR(length=36), ForeignKey(Origin.origin_ref), primary_key=True, nullable=False, index=True) - lease_ref = Column(CHAR(length=36), primary_key=True, nullable=False, index=True) lease_created = Column(DATETIME(), nullable=False) lease_expires = Column(DATETIME(), nullable=False) lease_updated = Column(DATETIME(), nullable=False) From 6049048bbf41e001e5d11ec8c2acb4064595bc15 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 11:24:40 +0100 Subject: [PATCH 05/15] fixed test --- .gitlab-ci.yml | 2 ++ test/main.py | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49e8c5f..20ae5f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,8 @@ build: test: image: python:3.10-slim-bullseye stage: test + variables: + DATABASE: sqlite:///../app/db.sqlite before_script: - pip install -r requirements.txt - pip install pytest httpx diff --git a/test/main.py b/test/main.py index aeb52e3..aa9d644 100644 --- a/test/main.py +++ b/test/main.py @@ -1,13 +1,11 @@ from starlette.testclient import TestClient -import importlib.util import sys -MODULE, PATH = 'main.app', '../app/main.py' +# add relative path to use packages as they were in the app/ dir +sys.path.append('../') +sys.path.append('../app') -spec = importlib.util.spec_from_file_location(MODULE, PATH) -main = importlib.util.module_from_spec(spec) -sys.modules[MODULE] = main -spec.loader.exec_module(main) +from app import main client = TestClient(main.app) From f539db5933b7156effede2666fac7e4bbe29df5e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:17:19 +0100 Subject: [PATCH 06/15] implemented db_init --- app/main.py | 3 ++- app/orm.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 15f0c35..5060361 100644 --- a/app/main.py +++ b/app/main.py @@ -22,7 +22,7 @@ from sqlalchemy.orm import sessionmaker from Crypto.PublicKey import RSA from Crypto.PublicKey.RSA import RsaKey -from orm import Origin, Lease +from orm import Origin, Lease, init as db_init logger = logging.getLogger() load_dotenv('../version.env') @@ -48,6 +48,7 @@ __details = dict( ) app, db = FastAPI(**__details), create_engine(url=str(getenv('DATABASE', 'sqlite:///db.sqlite'))) +db_init(db) TOKEN_EXPIRE_DELTA = relativedelta(hours=1) # days=1 LEASE_EXPIRE_DELTA = relativedelta(days=int(getenv('LEASE_EXPIRE_DAYS', 90))) diff --git a/app/orm.py b/app/orm.py index b7b05b7..bb806ae 100644 --- a/app/orm.py +++ b/app/orm.py @@ -1,6 +1,6 @@ import datetime -from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete +from sqlalchemy import Column, VARCHAR, CHAR, ForeignKey, DATETIME, UniqueConstraint, update, and_, delete, inspect from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.future import Engine from sqlalchemy.orm import sessionmaker @@ -102,3 +102,13 @@ class Lease(Base): deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) session.close() return deletions + + +def init(engine: Engine): + tables = [Origin, Lease] + db = inspect(engine) + session = sessionmaker(bind=engine)() + for table in tables: + if not db.dialect.has_table(engine.connect(), table.__tablename__): + session.execute(str(table.create_statement(engine))) + session.close() From 838e30458d4dc25a72326a4c8b34e23f029b3c65 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Fri, 23 Dec 2022 13:21:52 +0100 Subject: [PATCH 07/15] code styling --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 5060361..322e4a8 100644 --- a/app/main.py +++ b/app/main.py @@ -96,7 +96,7 @@ async def status(request: Request): @app.get('/-/origins') async def _origins(request: Request): - session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + session = sessionmaker(bind=db)() response = list(map(lambda x: jsonable_encoder(x), session.query(Origin).all())) session.close() return JSONResponse(response) @@ -104,7 +104,7 @@ async def _origins(request: Request): @app.get('/-/leases') async def _leases(request: Request): - session = sessionmaker(autocommit=False, autoflush=False, bind=db)() + session = sessionmaker(bind=db)() response = list(map(lambda x: jsonable_encoder(x), session.query(Lease).all())) session.close() return JSONResponse(response) From b5c64038cbde808d7e17182c6d27a898caa6fa5b Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 19:05:41 +0100 Subject: [PATCH 08/15] main.py - migrated merged changes from dataset to sqlalchemy --- app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main.py b/app/main.py index 4704ab1..84c5968 100644 --- a/app/main.py +++ b/app/main.py @@ -197,14 +197,14 @@ async def auth_v1_origin_update(request: Request): origin_ref = j['origin_ref'] logging.info(f'> [ update ]: {origin_ref}: {j}') - data = dict( + data = Origin( origin_ref=origin_ref, hostname=j['environment']['hostname'], guest_driver_version=j['environment']['guest_driver_version'], os_platform=j['environment']['os_platform'], os_version=j['environment']['os_version'], ) - db['origin'].upsert(data, ['origin_ref']) + Origin.create_or_update(db, data) response = { "environment": j['environment'], From 560b18b5c4ff9c2c143a17d2dcf92b0aeadf81db Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 19:57:58 +0100 Subject: [PATCH 09/15] orm.py - fixed not null column --- app/orm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/orm.py b/app/orm.py index bb806ae..7982b84 100644 --- a/app/orm.py +++ b/app/orm.py @@ -68,6 +68,8 @@ class Lease(Base): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() entity = session.query(Lease).filter(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).first() if entity is None: + if lease.lease_updated is None: + lease.lease_updated = lease.lease_created session.add(lease) else: values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) From 07f1e645531ac908a60f07bf51f7c29ca27b873e Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:05:55 +0100 Subject: [PATCH 10/15] fixes --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index 84c5968..b35db7d 100644 --- a/app/main.py +++ b/app/main.py @@ -334,7 +334,7 @@ async def leasing_v1_lessor_lease(request: Request): origin_ref = token['origin_ref'] - active_lease_list = list(map(lambda x: x['lease_ref'], db['lease'].find(origin_ref=origin_ref))) + active_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) logging.info(f'> [ leases ]: {origin_ref}: found {len(active_lease_list)} active leases') response = { From 85736c5ce4478d6b2efe71dc844bf23485864fc7 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:10:18 +0100 Subject: [PATCH 11/15] typos --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index b35db7d..52ec2ca 100644 --- a/app/main.py +++ b/app/main.py @@ -380,7 +380,7 @@ async def leasing_v1_lessor_lease_remove(request: Request): origin_ref = token['origin_ref'] released_lease_list = list(map(lambda x: x.lease_ref, Lease.find_by_origin_ref(db, origin_ref))) - deletions = Lease.ceanup(db, origin_ref) + deletions = Lease.cleanup(db, origin_ref) logging.info(f'> [ remove ]: {origin_ref}: removed {deletions} leases') response = { From 2a3e740964d029c9386409c57a7fe3f89eb951e9 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:19:23 +0100 Subject: [PATCH 12/15] added toc --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b014e5..37914cc 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. -## ToDo#'s +[[TOC]] + +## ToDo#'s / Roadmap - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) From 12bfd4c82aa82a04e249b290169ab0a121ce2d72 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:19:50 +0100 Subject: [PATCH 13/15] removed toc --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 37914cc..a61cf80 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,7 @@ Minimal Delegated License Service (DLS). This service can be used without internet connection. Only the clients need a connection to this service on configured port. -[[TOC]] - -## ToDo#'s / Roadmap +## ToDo's - provide `.deb` package (WIP) - migrate from `dataset` to `sqlalchemy` (WIP) From 1b34edfda6596408d6e97ee4062249caca46ec09 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:22:00 +0100 Subject: [PATCH 14/15] fixes --- app/orm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/orm.py b/app/orm.py index 7982b84..31fa64c 100644 --- a/app/orm.py +++ b/app/orm.py @@ -40,7 +40,7 @@ class Origin(Base): os_platform=origin.os_platform, os_version=origin.os_version, ) - session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(values)) + session.execute(update(Origin).where(Origin.origin_ref == origin.origin_ref).values(**values)) session.flush() session.close() @@ -73,7 +73,7 @@ class Lease(Base): session.add(lease) else: values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**values)) session.flush() session.close() @@ -95,7 +95,7 @@ class Lease(Base): def renew(engine: Engine, lease: "Lease", lease_expires: datetime.datetime, lease_updated: datetime.datetime): session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() values = dict(lease_expires=lease.lease_expires, lease_updated=lease.lease_updated) - session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(values)) + session.execute(update(Lease).where(and_(Lease.origin_ref == lease.origin_ref, Lease.lease_ref == lease.lease_ref)).values(**values)) session.close() @staticmethod From c38ed25a2fa22573dc0728a8d8682717be310ba6 Mon Sep 17 00:00:00 2001 From: Oscar Krause Date: Tue, 27 Dec 2022 20:28:09 +0100 Subject: [PATCH 15/15] fixes --- app/orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/orm.py b/app/orm.py index 31fa64c..26b869a 100644 --- a/app/orm.py +++ b/app/orm.py @@ -101,7 +101,7 @@ class Lease(Base): @staticmethod def cleanup(engine: Engine, origin_ref: str) -> int: session = sessionmaker(autocommit=True, autoflush=True, bind=engine)() - deletions = session.query(Lease).delete(Lease.origin_ref == origin_ref) + deletions = session.query(Lease).filter(Lease.origin_ref == origin_ref).delete() session.close() return deletions