feat: add Celery task: pull_repo

Signed-off-by: jingfelix <jingfelix@outlook.com>

server/model/schema.py

@@ -202,6 +202,9 @@ class RepoUser(Base):
         nullable=True,
         comment="哪一个application_id",
     )
+    repo_id = db.Column(
+        ObjID(12), ForeignKey("repo.id"), nullable=True, comment="属于哪一个项目"
+    )
     bind_user_id = db.Column(
         ObjID(12), ForeignKey("bind_user.id"), nullable=True, comment="项目协作者"
     )

server/model/team.py

@@ -122,6 +122,33 @@ def set_team_member(team_id, code_user_id, im_user_id):
     db.session.commit()
 
 
+def add_team_member(team_id, code_user_id):
+    """Add a team member.
+    Args:
+        team_id (str): Team ID.
+        code_user_id (str): BindUser ID.
+    """
+    # 检查是否已经存在
+    if (
+        TeamMember.query.filter_by(
+            team_id=team_id,
+            code_user_id=code_user_id,
+            status=0,
+        ).first()
+        is not None
+    ):
+        return
+
+    new_team_member = TeamMember(
+        id=ObjID.new_id(),
+        team_id=team_id,
+        code_user_id=code_user_id,
+        im_user_id=None,
+    )
+    db.session.add(new_team_member)
+    db.session.commit()
+
+
 def create_team(app_info: dict) -> Team:
     """Create a team.
 

server/routes/github.py

@@ -4,6 +4,7 @@
 from app import app
 from flask import Blueprint, make_response, redirect, request, session
 from model.team import create_code_application, create_team
+from tasks.github import pull_repo
 from utils.auth import authenticated
 from utils.github.application import verify_github_signature
 from utils.github.bot import BaseGitHubApp
@@ -45,9 +46,15 @@ def github_install():
 
     except Exception as e:
         # 返回错误信息
+        app.logger.error(e)
         app_info = str(e)
 
-    # TODO: 加入后台任务
+    # 在后台任务中拉取仓库
+    results = pull_repo.delay(
+        org_name=app_info["account"]["login"],
+        installation_id=installation_id,
+        application_id=code_application.id,
+    )
 
     return make_response(
         """

server/tasks/github.py

@@ -1,38 +1,89 @@
-import os
-
 from celery_app import celery
-from model.schema import db
-from utils.github.application import get_installation_token, get_jwt
-from utils.github.organization import get_org_members
+from model.schema import BindUser, ObjID, Repo, RepoUser, User, db
+from model.team import add_team_member
+from utils.github.organization import GitHubAppOrg
+from utils.user import create_github_user
 
 
 @celery.task()
-def pull_repo(org_name: str, installation_id: str, application_id: str):
-    """Pull repo from GitHub, build Repo and RepoUser."""
-    # 获取 jwt 和 installation_token
+def pull_repo(org_name: str, installation_id: str, application_id: str, team_id: str):
+    """Pull repo from GitHub, build Repo and RepoUser.
 
-    installation_token: str = get_installation_token(
-        get_jwt(
-            os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH"),
-            os.environ.get("GITHUB_APP_ID"),
-        ),
-        installation_id,
-    )  # TODO: 这里有一个遗留问题：installation_token 有一小时的期限
-    # 解决方案是在 utils.github.application 中增加一个类，用于统一管理 jwt 和 installation_token
+    Args:
+        org_name: GitHub organization name.
+        installation_id: GitHub App installation id.
+        application_id: Code application id.
+    """
+    # 获取 jwt 和 installation_token
 
-    if installation_token is None:
-        raise Exception("Failed to get installation token.")  # TODO: 统一处理 celery 报错？
+    github_app = GitHubAppOrg(installation_id)
 
-    # 拉取所有组织成员，创建 BindUser
-    members = get_org_members(org_name, installation_token)
+    # 拉取所有组织成员，创建 User 和 BindUser
+    members = github_app.get_org_members(org_name)
     if members is None or not isinstance(members, list):
         raise Exception("Failed to get org members.")
 
     for member in members:
-        # 检查是否已经存在（其实只有一个人会重复？）
-        pass
+        # 已存在的用户不会重复创建
+        _, new_bind_user_id = create_github_user(
+            github_id=member["id"],
+            name=member["login"],
+            email=member.get("email", None),
+            avatar=member["avatar_url"],
+            access_token=None,
+            application_id=application_id,
+            extra={},
+        )
+
+        add_team_member(team_id, new_bind_user_id)
 
     # 拉取所有组织仓库，创建 Repo
-    # 给每个仓库创建 RepoUser
+    repos = github_app.get_org_repos(org_name)
+    try:
+        for repo in repos:
+            # 检查是否已经存在
+            if Repo.query.filter_by(repo_id=repo["id"]).first() is not None:
+                continue
+
+            new_repo = Repo(
+                id=ObjID.new_id(),
+                application_id=application_id,
+                owner_bind_id=None,  # TODO: 暂定不填写
+                repo_id=repo["id"],
+                description=repo["description"],
+            )
+            db.session.add(new_repo)
+            db.session.flush()
+
+            # 拉取仓库成员，创建 RepoUser
+            repo_users = github_app.get_repo_collaborators(repo["name"], org_name)
+            print(repo_users)
+
+            # 检查是否有 bind_user
+            for repo_user in repo_users:
+                bind_user = (
+                    db.session.query(BindUser)
+                    .filter(
+                        User.unionid == repo_user["id"],
+                        BindUser.platform == "github",
+                        BindUser.application_id == application_id,
+                        BindUser.user_id == User.id,
+                    )
+                    .first()
+                )
+                if bind_user is None:
+                    continue
+
+                new_repo_user = RepoUser(
+                    id=ObjID.new_id(),
+                    application_id=application_id,
+                    repo_id=new_repo.id,
+                    bind_user_id=bind_user.id,
+                )
+                db.session.add(new_repo_user)
+                db.session.commit()
 
-    pass
+        db.session.commit()
+    except Exception as e:
+        db.session.rollback()
+        raise e

server/utils/github/organization.py

@@ -1,30 +1,55 @@
-import httpx
-from app import app
+from utils.github.bot import BaseGitHubApp
 
 
-def get_org_members(org_name: str, installation_token: str) -> list | None:
-    """Get a list of members of an organization.
+class GitHubAppOrg(BaseGitHubApp):
+    def __init__(self, installation_id: str):
+        super().__init__(installation_id)
 
-    Args:
-        org_name (str): The name of the organization.
-        installation_token (str): The installation token for the GitHub App.
+    def get_org_repos(self, org_name: str) -> list | None:
+        """Get org repos.
 
-    Returns:
-        list | None: A list of members of the organization.
-    https://docs.github.com/zh/rest/orgs/members?apiVersion=2022-11-28#list-organization-members
-    """
-    with httpx.Client() as client:
-        response = client.get(
-            f"https://api.github.com/orgs/{org_name}/members",
-            headers={
-                "Accept": "application/vnd.github+json",
-                "Authorization": f"token {installation_token}",
-                "X-GitHub-Api-Version": "2022-11-28",
-            },
+        Args:
+            org_name (str): The name of the org.
+
+        Returns:
+            list: The org repos.
+        https://docs.github.com/zh/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories
+        """
+
+        return self.base_github_rest_api(
+            f"https://api.github.com/orgs/{org_name}/repos",
+            auth_type="install_token",
         )
 
-        if response.status_code == 200:
-            return response.json()
-        else:
-            app.logger.debug(f"Failed to get org members. {response.text}")
-            return None
+    def get_repo_collaborators(self, repo_name: str, owner_name: str) -> list | None:
+        """Get repo collaborators.
+
+        Args:
+            repo_name (str): The name of the repo.
+            owner_name (str): The name of the owner.
+
+        Returns:
+            list: The repo collaborators.
+        https://docs.github.com/zh/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators
+        """
+
+        return self.base_github_rest_api(
+            f"https://api.github.com/repos/{owner_name}/{repo_name}/collaborators",
+            auth_type="install_token",
+        )
+
+    def get_org_members(self, org_name: str) -> list | None:
+        """Get a list of members of an organization.
+
+        Args:
+            org_name (str): The name of the organization.
+
+        Returns:
+            list | None: A list of members of the organization.
+        https://docs.github.com/zh/rest/orgs/members?apiVersion=2022-11-28#list-organization-members
+        """
+
+        return self.base_github_rest_api(
+            f"https://api.github.com/orgs/{org_name}/members",
+            auth_type="install_token",
+        )

server/utils/user.py

@@ -31,7 +31,7 @@ def register(code: str) -> str | None:
 
     email = get_email(access_token)
 
-    new_user_id = create_github_user(
+    new_user_id, _ = create_github_user(
         github_id=github_id,
         name=user_info.get("name", None),
         email=email,
@@ -49,8 +49,9 @@ def create_github_user(
     email: str,
     avatar: str,
     access_token: str = None,
+    application_id: str = None,
     extra: dict = {},
-) -> str:
+) -> (str, str):
     """Create a GitHub user.
 
     Args:
@@ -76,9 +77,12 @@ def create_github_user(
             # 刷新 access_token
             if access_token is not None:
                 bind_user.access_token = access_token
-                db.session.commit()
 
-            return user.id
+            if application_id is not None:
+                bind_user.application_id = application_id
+
+            db.session.commit()
+            return user.id, bind_user.id
 
     new_user = User(
         id=ObjID.new_id(),
@@ -100,11 +104,12 @@ def create_github_user(
         name=name,
         avatar=avatar,
         access_token=access_token,
+        application_id=application_id,
         extra=extra.get("oauth_info", None),
     )
 
     db.session.add(new_bind_user)
 
     db.session.commit()
 
-    return new_user.id
+    return new_user.id, new_bind_user.id