dropped db, works with plain file structure now
All checks were successful
continuous-integration/drone/push Build is passing

Signed-off-by: Hipstercat <tasty@hipstercat.fr>
This commit is contained in:
Amazed 2021-12-22 23:25:46 +01:00
parent cb2c0a84aa
commit 7b74f1240f
Signed by: hipstercat
GPG Key ID: BF42C937290F5641
9 changed files with 57 additions and 89 deletions

View File

@ -5,3 +5,4 @@ config = None
with open("config.json", "r") as fp: with open("config.json", "r") as fp:
print("loading config") print("loading config")
config = json.load(fp) config = json.load(fp)
config["upload_path"] = "blobs/"

View File

@ -1,12 +0,0 @@
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
SQLALCHEMY_DATABASE_URL = "sqlite:///./database.sqlite"
engine = create_engine(
SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()

View File

@ -1,10 +0,0 @@
from .database import SessionLocal
# Dependency
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()

View File

@ -1,12 +1,8 @@
from fastapi import Depends, FastAPI from fastapi import FastAPI
from .routers import maps from app.routers import maps
import app.models
from .database import engine
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
app.models.Base.metadata.create_all(bind=engine)
app = FastAPI( app = FastAPI(
title="Giants: Citizen Kabuto map API", title="Giants: Citizen Kabuto map API",
description="API to upload and download maps for Giants: Citizen Kabuto", description="API to upload and download maps for Giants: Citizen Kabuto",

View File

@ -1,15 +0,0 @@
import datetime
from sqlalchemy import Column, Integer, String, DateTime
# from sqlalchemy.orm import relationship
from .database import Base
class Map(Base):
__tablename__ = "maps"
crc = Column(Integer, primary_key=True, index=True)
name = Column(String)
size = Column(Integer)
upload_date = Column(DateTime, default=datetime.datetime.utcnow)
filename = Column(String)

View File

@ -1,31 +1,30 @@
import base64 import base64
import struct
import io import io
import os.path
import pathlib
import zipfile import zipfile
from typing import List, Optional from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query from fastapi import APIRouter, HTTPException, Query
from sqlalchemy.orm import Session
from ..dependencies import get_db
from ..schemas import * from ..schemas import *
from ..utils import crc32, map_model_to_out_schema from ..utils import crc32, read_all_maps, map_file_to_dict
from ..models import *
from ..config import config from ..config import config
import re
router = APIRouter() router = APIRouter()
MAX_UPLOAD_SIZE = config["max_file_size"] MAX_UPLOAD_SIZE = config["max_file_size"]
MAPS = read_all_maps()
@router.get("/maps", tags=["maps"], response_model=List[MapOut]) @router.get("/maps", tags=["maps"], response_model=List[MapOut])
async def get_all_maps(db: Session = Depends(get_db)): async def get_all_maps():
return [map_model_to_out_schema(m) for m in db.query(Map).all()] return [MapOut(**d) for d in MAPS]
@router.get("/map", tags=["maps"], response_model=MapOut) @router.get("/map", tags=["maps"], response_model=MapOut)
async def get_map_by_crc(human_crc: Optional[str] = Query(None, min_length=8, max_length=8), async def get_map_by_crc(human_crc: Optional[str] = Query(None, min_length=8, max_length=8),
crc: Optional[int] = Query(None), crc: Optional[int] = Query(None)):
db: Session = Depends(get_db)):
if not human_crc and not crc: if not human_crc and not crc:
raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both") raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both")
@ -33,20 +32,20 @@ async def get_map_by_crc(human_crc: Optional[str] = Query(None, min_length=8, ma
raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both") raise HTTPException(status_code=400, detail="Please use crc or human_crc but not both")
if human_crc: if human_crc:
crc_int = struct.unpack(">L", bytes.fromhex(human_crc))[0] maps = [gmap for gmap in MAPS if gmap["crc_human"] == human_crc.upper()]
else: else:
crc_int = crc maps = [gmap for gmap in MAPS if gmap["crc"] == crc]
existing_map = db.query(Map).filter(Map.crc == crc_int).first() if not maps:
if not existing_map:
raise HTTPException(status_code=404, detail="Map not found") raise HTTPException(status_code=404, detail="Map not found")
return map_model_to_out_schema(existing_map) return MapOut(**maps[0])
@router.post("/maps", tags=["maps"], response_model=MapOut) @router.post("/maps", tags=["maps"], response_model=MapOut)
async def upload_map(map_in: MapIn, db: Session = Depends(get_db)): async def upload_map(map_in: MapIn):
allowed_name = re.compile("[a-zA-Z-.\d()[] ]+.gck")
filename = map_in.name filename = map_in.name
if not filename.lower().endswith(".gck"): if not allowed_name.fullmatch(filename):
raise HTTPException(status_code=400, detail="Invalid file") raise HTTPException(status_code=400, detail="Invalid filename")
map_bytes = base64.b64decode(map_in.b64_data.encode("utf8")) map_bytes = base64.b64decode(map_in.b64_data.encode("utf8"))
if len(map_bytes) > MAX_UPLOAD_SIZE: if len(map_bytes) > MAX_UPLOAD_SIZE:
@ -59,21 +58,17 @@ async def upload_map(map_in: MapIn, db: Session = Depends(get_db)):
raise HTTPException(status_code=400, detail="File is not a valid map") raise HTTPException(status_code=400, detail="File is not a valid map")
crc = crc32(map_bytes) crc = crc32(map_bytes)
existing_map = db.query(Map).filter(Map.crc == crc).first() existing_map = [gmap for gmap in MAPS if gmap["crc"] == crc]
if existing_map: if existing_map:
return map_model_to_out_schema(existing_map) return MapOut(**existing_map[0])
else: else:
uploaded_filename = "%s.gck" % crc uploaded_filename = f"{filename}.gck"
with open("%s%s" % (config["upload_path"], uploaded_filename), "wb") as fp: i = 0
while os.path.exists(f"{config['upload_path']}{uploaded_filename}"):
uploaded_filename = f"{filename}-{i}.gck"
i += 1
with open(f"{config['upload_path']}{uploaded_filename}", "wb") as fp:
fp.write(map_bytes) fp.write(map_bytes)
gmap = map_file_to_dict(pathlib.Path(f"{config['upload_path']}{uploaded_filename}"))
uploaded_map = Map() MAPS.append(gmap)
uploaded_map.crc = crc return MapOut(**gmap)
uploaded_map.name = map_in.name
uploaded_map.filename = uploaded_filename
uploaded_map.size = len(map_bytes)
db.add(uploaded_map)
db.commit()
db.refresh(uploaded_map)
return map_model_to_out_schema(uploaded_map)

View File

@ -10,9 +10,6 @@ class MapOut(BaseModel):
upload_date: datetime.datetime upload_date: datetime.datetime
blob_location: str blob_location: str
class Config:
orm_mode = True
class MapIn(BaseModel): class MapIn(BaseModel):
name: str name: str

View File

@ -1,17 +1,34 @@
import zlib import zlib
from .models import Map import pathlib
from .schemas import MapOut from typing import List
import copy
from .config import config from .config import config
import os
import datetime
def crc32(bytes_in: bytes) -> int: def crc32(bytes_in: bytes) -> int:
return zlib.crc32(bytes_in, 0) & 0xffffffff return zlib.crc32(bytes_in, 0) & 0xffffffff
def map_model_to_out_schema(map_in: Map) -> MapOut: def map_file_to_dict(map_file: pathlib.Path) -> dict:
d = copy.deepcopy(map_in.__dict__) with open(map_file, "rb") as fp:
d["crc_human"] = hex(map_in.crc)[2:].upper() content = fp.read()
d["blob_location"] = "%s/%s%s" % (config["base_url"], config["upload_path"], map_in.filename) crc = crc32(content)
map_out = MapOut(**d)
return map_out return {
"crc": crc,
"crc_human": hex(crc)[2:].upper(),
"name": map_file.name,
"size": os.stat(map_file).st_size,
"upload_date": datetime.datetime.fromtimestamp(os.stat(map_file).st_ctime),
"blob_location": f"{config['base_url']}/{config['upload_path']}{map_file.name}"
}
def read_all_maps() -> List[dict]:
print("reading all maps")
maps = []
for file in os.listdir(config["upload_path"]):
maps.append(map_file_to_dict(pathlib.Path(config["upload_path"] + "/" + file)))
return maps

View File

@ -1,5 +1,4 @@
{ {
"base_url": "https://gckmaps.hipstercat.fr", "base_url": "https://gckmaps.hipstercat.fr",
"upload_path": "blobs/",
"max_file_size": 104857600 "max_file_size": 104857600
} }