Skip to content

Commit

Permalink
fix: oom
Browse files Browse the repository at this point in the history
  • Loading branch information
z0z0r4 committed Jan 1, 2025
1 parent 2aa227d commit 16a86a0
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 20 deletions.
18 changes: 13 additions & 5 deletions database/mongodb.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
from odmantic import AIOEngine, SyncEngine
from motor.motor_asyncio import AsyncIOMotorClient
from pymongo import MongoClient
from pymongo.database import Database

from config import Config
from models.database.curseforge import Mod, File, Fingerprint
from models.database.modrinth import Project, Version, File as ModrinthFile
from models.database.file_cdn import File as CDNFile

_mongodb_config = Config.load().mongodb

Expand All @@ -22,9 +19,20 @@ def init_mongodb_syncengine() -> SyncEngine:
if _mongodb_config.auth
else f"mongodb://{_mongodb_config.host}:{_mongodb_config.port}"
),
database="mcim_backend",
database=_mongodb_config.database,
)
return sync_mongo_engine


def init_mongodb_raw_client() -> Database:
global raw_mongo_client
raw_mongo_client = MongoClient(
f"mongodb://{_mongodb_config.user}:{_mongodb_config.password}@{_mongodb_config.host}:{_mongodb_config.port}"
if _mongodb_config.auth
else f"mongodb://{_mongodb_config.host}:{_mongodb_config.port}"
)[_mongodb_config.database]
return raw_mongo_client


sync_mongo_engine: SyncEngine = init_mongodb_syncengine()
raw_mongo_client: Database = init_mongodb_raw_client()
22 changes: 7 additions & 15 deletions sync/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import datetime
import time

from database.mongodb import sync_mongo_engine
from database.mongodb import sync_mongo_engine, raw_mongo_client
from utils.loger import log
from config import Config
from models.database.curseforge import Mod
Expand Down Expand Up @@ -316,22 +316,14 @@ def check_new_modids(modids: List[int]) -> List[int]:
"""
返回对应的 modids
"""
# in chunk
result: List[int] = []
for i in range(0, len(modids), CURSEFORGE_LIMIT_SIZE):
chunk = modids[i : i + CURSEFORGE_LIMIT_SIZE]
result.extend([mod.id for mod in sync_mongo_engine.find(Mod, query.not_in(Mod.id, chunk))])
return result
find_result = raw_mongo_client["curseforge_mods"].find({"_id": {"$in": modids}}, {"_id": 1})
found_modids = [mod["_id"] for mod in find_result]
return list(set(modids) - set(found_modids))

def check_new_project_ids(project_ids: List[str]) -> List[str]:
"""
返回对应的 project_ids
"""
# in chunk
result: List[str] = []
for i in range(0, len(project_ids), MODRINTH_LIMIT_SIZE):
chunk = project_ids[i : i + MODRINTH_LIMIT_SIZE]
result.extend(
[project.id for project in sync_mongo_engine.find(Project, query.not_in(Project.id, chunk))]
)
return result
find_result = raw_mongo_client["modrinth_projects"].find({"_id": {"$in": project_ids}}, {"_id": 1})
found_project_ids = [project["_id"] for project in find_result]
return list(set(project_ids) - set(found_project_ids))

0 comments on commit 16a86a0

Please sign in to comment.