dropped db, works with plain file structure now
All checks were successful
continuous-integration/drone/push Build is passing

Signed-off-by: Hipstercat <tasty@hipstercat.fr>
This commit is contained in:
Amazed 2021-12-22 23:25:46 +01:00
parent cb2c0a84aa
commit 7b74f1240f
Signed by: hipstercat
GPG Key ID: BF42C937290F5641
9 changed files with 57 additions and 89 deletions

View File

@ -5,3 +5,4 @@ config = None
with open("config.json", "r") as fp:
print("loading config")
config = json.load(fp)
config["upload_path"] = "blobs/"

View File

@ -1,12 +0,0 @@
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
SQLALCHEMY_DATABASE_URL = "sqlite:///./database.sqlite"
engine = create_engine(
SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()

View File

@ -1,10 +0,0 @@
from .database import SessionLocal
# Dependency
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()

View File

@ -1,12 +1,8 @@
from fastapi import Depends, FastAPI
from .routers import maps
import app.models
from .database import engine
from fastapi import FastAPI
from app.routers import maps
from fastapi.staticfiles import StaticFiles
app.models.Base.metadata.create_all(bind=engine)
app = FastAPI(
title="Giants: Citizen Kabuto map API",
description="API to upload and download maps for Giants: Citizen Kabuto",

View File

@ -1,15 +0,0 @@
import datetime
from sqlalchemy import Column, Integer, String, DateTime
# from sqlalchemy.orm import relationship
from .database import Base
class Map(Base):
__tablename__ = "maps"
crc = Column(Integer, primary_key=True, index=True)
name = Column(String)
size = Column(Integer)
upload_date = Column(DateTime, default=datetime.datetime.utcnow)
filename = Column(String)

View File

@ -1,31 +1,30 @@
import base64
import struct
import io
import os.path
import pathlib
import zipfile
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from ..dependencies import get_db
from fastapi import APIRouter, HTTPException, Query
from ..schemas import *
from ..utils import crc32, map_model_to_out_schema
from ..models import *
from ..utils import crc32, read_all_maps, map_file_to_dict
from ..config import config
import re
router = APIRouter()
MAX_UPLOAD_SIZE = config["max_file_size"]
MAPS = read_all_maps()
@router.get("/maps", tags=["maps"], response_model=List[MapOut])
async def get_all_maps(db: Session = Depends(get_db)):
return [map_model_to_out_schema(m) for m in db.query(Map).all()]
async def get_all_maps():
return [MapOut(**d) for d in MAPS]
@router.get("/map", tags=["maps"], response_model=MapOut)
async def get_map_by_crc(human_crc: Optional[str] = Query(None, min_length=8, max_length=8),
crc: Optional[int] = Query(None),
db: Session = Depends(get_db)):
crc: Optional[int] = Query(None)):
if not human_crc and not crc:
raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both")
@ -33,20 +32,20 @@ async def get_map_by_crc(human_crc: Optional[str] = Query(None, min_length=8, ma
raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both")
if human_crc:
crc_int = struct.unpack(">L", bytes.fromhex(human_crc))[0]
maps = [gmap for gmap in MAPS if gmap["crc_human"] == human_crc.upper()]
else:
crc_int = crc
existing_map = db.query(Map).filter(Map.crc == crc_int).first()
if not existing_map:
maps = [gmap for gmap in MAPS if gmap["crc"] == crc]
if not maps:
raise HTTPException(status_code=404, detail="Map not found")
return map_model_to_out_schema(existing_map)
return MapOut(**maps[0])
@router.post("/maps", tags=["maps"], response_model=MapOut)
async def upload_map(map_in: MapIn, db: Session = Depends(get_db)):
async def upload_map(map_in: MapIn):
allowed_name = re.compile("[a-zA-Z-.\d()[] ]+.gck")
filename = map_in.name
if not filename.lower().endswith(".gck"):
raise HTTPException(status_code=400, detail="Invalid file")
if not allowed_name.fullmatch(filename):
raise HTTPException(status_code=400, detail="Invalid filename")
map_bytes = base64.b64decode(map_in.b64_data.encode("utf8"))
if len(map_bytes) > MAX_UPLOAD_SIZE:
@ -59,21 +58,17 @@ async def upload_map(map_in: MapIn, db: Session = Depends(get_db)):
raise HTTPException(status_code=400, detail="File is not a valid map")
crc = crc32(map_bytes)
existing_map = db.query(Map).filter(Map.crc == crc).first()
existing_map = [gmap for gmap in MAPS if gmap["crc"] == crc]
if existing_map:
return map_model_to_out_schema(existing_map)
return MapOut(**existing_map[0])
else:
uploaded_filename = "%s.gck" % crc
with open("%s%s" % (config["upload_path"], uploaded_filename), "wb") as fp:
uploaded_filename = f"{filename}.gck"
i = 0
while os.path.exists(f"{config['upload_path']}{uploaded_filename}"):
uploaded_filename = f"{filename}-{i}.gck"
i += 1
with open(f"{config['upload_path']}{uploaded_filename}", "wb") as fp:
fp.write(map_bytes)
uploaded_map = Map()
uploaded_map.crc = crc
uploaded_map.name = map_in.name
uploaded_map.filename = uploaded_filename
uploaded_map.size = len(map_bytes)
db.add(uploaded_map)
db.commit()
db.refresh(uploaded_map)
return map_model_to_out_schema(uploaded_map)
gmap = map_file_to_dict(pathlib.Path(f"{config['upload_path']}{uploaded_filename}"))
MAPS.append(gmap)
return MapOut(**gmap)

View File

@ -10,9 +10,6 @@ class MapOut(BaseModel):
upload_date: datetime.datetime
blob_location: str
class Config:
orm_mode = True
class MapIn(BaseModel):
name: str

View File

@ -1,17 +1,34 @@
import zlib
from .models import Map
from .schemas import MapOut
import copy
import pathlib
from typing import List
from .config import config
import os
import datetime
def crc32(bytes_in: bytes) -> int:
return zlib.crc32(bytes_in, 0) & 0xffffffff
def map_model_to_out_schema(map_in: Map) -> MapOut:
d = copy.deepcopy(map_in.__dict__)
d["crc_human"] = hex(map_in.crc)[2:].upper()
d["blob_location"] = "%s/%s%s" % (config["base_url"], config["upload_path"], map_in.filename)
map_out = MapOut(**d)
return map_out
def map_file_to_dict(map_file: pathlib.Path) -> dict:
with open(map_file, "rb") as fp:
content = fp.read()
crc = crc32(content)
return {
"crc": crc,
"crc_human": hex(crc)[2:].upper(),
"name": map_file.name,
"size": os.stat(map_file).st_size,
"upload_date": datetime.datetime.fromtimestamp(os.stat(map_file).st_ctime),
"blob_location": f"{config['base_url']}/{config['upload_path']}{map_file.name}"
}
def read_all_maps() -> List[dict]:
print("reading all maps")
maps = []
for file in os.listdir(config["upload_path"]):
maps.append(map_file_to_dict(pathlib.Path(config["upload_path"] + "/" + file)))
return maps

View File

@ -1,5 +1,4 @@
{
"base_url": "https://gckmaps.hipstercat.fr",
"upload_path": "blobs/",
"max_file_size": 104857600
}