Merge pull request #2590 from chaoss/gitlab-changes

Implement ability to add gitlab repos from frontend

augur/api/routes/dei.py

@@ -52,7 +52,7 @@ def dei_track_repo(application: ClientApplication):
         return jsonify({"status": "Repo already exists"})
 
     frontend_repo_group: RepoGroup = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first()
-    repo_id = Repo.insert(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="")
+    repo_id = Repo.insert_github_repo(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="")
     if not repo_id:
         return jsonify({"status": "Error adding repo"})
 

augur/api/routes/user.py

@@ -227,7 +227,7 @@ def add_user_repo():
     repo = request.args.get("repo_url")
     group_name = request.args.get("group_name")
 
-    result = current_user.add_repo(group_name, repo)
+    result = current_user.add_github_repo(group_name, repo)
 
     return jsonify(result[1])
 
@@ -260,7 +260,7 @@ def add_user_org():
     org = request.args.get("org_url")
     group_name = request.args.get("group_name")
 
-    result = current_user.add_org(group_name, org)
+    result = current_user.add_github_org(group_name, org)
 
     return jsonify(result[1])
 

augur/api/view/api.py

@@ -102,7 +102,18 @@ def av_add_user_repo():
                 if rg_obj:
                     # add the orgs repos to the group
                     add_existing_org_to_group(session, current_user.user_id, group, rg_obj.repo_group_id)
-
+
+            # matches https://gitlab.com/{org}/{repo}/ or http://gitlab.com/{org}/{repo}
+            elif Repo.parse_gitlab_repo_url(url)[0]:
+
+                org_name, repo_name = Repo.parse_github_repo_url(url)
+                repo_git = f"https://gitlab.com/{org_name}/{repo_name}"
+
+                # TODO: gitlab ensure the whole repo git is inserted so it can be found here
+                repo_obj = Repo.get_by_repo_git(session, repo_git)
+                if repo_obj:
+                    add_existing_repo_to_group(session, current_user.user_id, group, repo_obj.repo_id)
+
             else:
                 invalid_urls.append(url)
 

augur/application/db/models/augur_data.py

@@ -929,6 +929,44 @@ def is_valid_github_repo(gh_session, url: str) -> bool:
                 return False, {"status": f"Github Error: {data['message']}"}
 
             return True, {"status": "Valid repo", "repo_type": data["owner"]["type"]}
+
+    @staticmethod
+    def is_valid_gitlab_repo(gl_session, url: str) -> bool:
+        """Determine whether a GitLab repo URL is valid.
+
+        Args:
+            gl_session: GitLab session object with API key
+            url: Repository URL
+
+        Returns:
+            True if repo URL is valid, False otherwise
+        """
+        from augur.tasks.github.util.github_paginator import hit_api
+
+        REPO_ENDPOINT = "https://gitlab.com/api/v4/projects/{}/"
+
+        owner, repo = Repo.parse_gitlab_repo_url(url)
+        if not owner or not repo:
+            return False, {"status": "Invalid repo URL"}
+
+        # Encode namespace and project name for the API request
+        project_identifier = f"{owner}%2F{repo}"
+        url = REPO_ENDPOINT.format(project_identifier)
+
+        attempts = 0
+        while attempts < 10:
+            response = hit_api(gl_session.oauths, url, logger)
+
+            if response.status_code == 404:
+                return False, {"status": "Invalid repo"}
+
+            if response.status_code == 200:
+                return True, {"status": "Valid repo"}
+
+            attempts += 1
+
+        return False, {"status": "Failed to validate repo after multiple attempts"}
+
 
     @staticmethod
     def parse_github_repo_url(url: str) -> tuple:
@@ -948,6 +986,29 @@ def parse_github_repo_url(url: str) -> tuple:
 
         capturing_groups = result.groups()
 
+        owner = capturing_groups[0]
+        repo = capturing_groups[1]
+
+        return owner, repo
+
+    @staticmethod
+    def parse_gitlab_repo_url(url: str) -> tuple:
+        """ Gets the owner and repo from a gitlab url.
+
+        Args:
+            url: Gitlab url
+
+        Returns:
+            Tuple of owner and repo. Or a tuple of None and None if the url is invalid.
+        """
+
+        result = re.search(r"https?:\/\/gitlab\.com\/([A-Za-z0-9 \- _]+)\/([A-Za-z0-9 \- _ \.]+)(.git)?\/?$", url)
+
+        if not result:
+            return None, None
+
+        capturing_groups = result.groups()
+
 
         owner = capturing_groups[0]
         repo = capturing_groups[1]
@@ -974,7 +1035,54 @@ def parse_github_org_url(url):
         return result.groups()[0]
 
     @staticmethod
-    def insert(session, url: str, repo_group_id: int, tool_source, repo_type):
+    def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source):
+        """Add a repo to the repo table.
+
+        Args:
+            url: repo url
+            repo_group_id: group to assign repo to
+
+        Note:
+            If repo row exists then it will update the repo_group_id if param repo_group_id is not a default. If it does not exist is will simply insert the repo.
+        """
+
+        if not isinstance(url, str) or not isinstance(repo_group_id, int) or not isinstance(tool_source, str):
+            return None
+
+        if not RepoGroup.is_valid_repo_group_id(session, repo_group_id):
+            return None
+
+        if url.endswith("/"):
+            url = url[:-1]
+
+        url = url.lower()
+
+        owner, repo = Repo.parse_gitlab_repo_url(url)
+        if not owner or not repo:
+            return None
+
+        repo_data = {
+            "repo_group_id": repo_group_id,
+            "repo_git": url,
+            "repo_path": f"gitlab.com/{owner}/",
+            "repo_name": repo,
+            "repo_type": None,
+            "tool_source": tool_source,
+            "tool_version": "1.0",
+            "data_source": "Git"
+        }
+
+        repo_unique = ["repo_git"]
+        return_columns = ["repo_id"]
+        result = session.insert_data(repo_data, Repo, repo_unique, return_columns, on_conflict_update=False)
+
+        if not result:
+            return None
+
+        return result[0]["repo_id"]
+
+    @staticmethod
+    def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_type):
         """Add a repo to the repo table.
 
         Args:

augur/application/db/models/augur_operations.py

@@ -449,17 +449,30 @@ def remove_group(self, group_name):
 
         return result
 
-    def add_repo(self, group_name, repo_url):
+    def add_github_repo(self, group_name, repo_url):
 
         from augur.tasks.github.util.github_task_session import GithubTaskSession
         from augur.tasks.github.util.github_api_key_handler import NoValidKeysError
         try:
             with GithubTaskSession(logger) as session:
-                result = UserRepo.add(session, repo_url, self.user_id, group_name)
+                result = UserRepo.add_github_repo(session, repo_url, self.user_id, group_name)
         except NoValidKeysError:
             return False, {"status": "No valid keys"}
 
         return result
+
+    def add_gitlab_repo(self, group_name, repo_url):
+
+        from augur.tasks.gitlab.gitlab_task_session import GitlabTaskSession
+        from augur.tasks.github.util.github_api_key_handler import NoValidKeysError
+        try:
+            with GitlabTaskSession(logger) as session:
+                result = UserRepo.add_gitlab_repo(session, repo_url, self.user_id, group_name)
+        except NoValidKeysError:
+            return False, {"status": "No valid keys"}
+
+        return result
+
 
     def remove_repo(self, group_name, repo_id):
 
@@ -468,14 +481,14 @@ def remove_repo(self, group_name, repo_id):
 
         return result
 
-    def add_org(self, group_name, org_url):
+    def add_github_org(self, group_name, org_url):
 
         from augur.tasks.github.util.github_task_session import GithubTaskSession
         from augur.tasks.github.util.github_api_key_handler import NoValidKeysError
 
         try:
             with GithubTaskSession(logger) as session:
-                result = UserRepo.add_org_repos(session, org_url, self.user_id, group_name)
+                result = UserRepo.add_github_org_repos(session, org_url, self.user_id, group_name)
         except NoValidKeysError:
             return False, {"status": "No valid keys"}
 
@@ -769,9 +782,69 @@ def insert(session, repo_id: int, group_id:int = 1) -> bool:
             return False
 
         return data[0]["group_id"] == group_id and data[0]["repo_id"] == repo_id
+
+    @staticmethod
+    def add_gitlab_repo(session, url: List[str], user_id: int, group_name=None, group_id=None, from_org_list=False, repo_group_id=None) -> dict:
+        """Add repo to the user repo table
+
+        Args:
+            urls: list of repo urls
+            user_id: id of user_id from users table
+            group_name: name of group to add repo to.
+            group_id: id of the group
+            valid_repo: boolean that indicates whether the repo has already been validated
+
+        Note:
+            Either the group_name or group_id can be passed not both
+
+        Returns:
+            Dict that contains the key "status" and additional useful data
+        """
+
+        if group_name and group_id:
+            return False, {"status": "Pass only the group name or group id not both"}
+
+        if not group_name and not group_id:
+            return False, {"status": "Need group name or group id to add a repo"}
+
+        if group_id is None:
+
+            group_id = UserGroup.convert_group_name_to_id(session, user_id, group_name)
+            if group_id is None:
+                return False, {"status": "Invalid group name"}
+
+        if not from_org_list:
+            result = Repo.is_valid_gitlab_repo(session, url)
+            if not result[0]:
+                return False, {"status": result[1]["status"], "repo_url": url}
+
+        # if no repo_group_id is passed then assign the repo to the frontend repo group
+        if repo_group_id is None:
+
+            frontend_repo_group = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first()
+            if not frontend_repo_group:
+                return False, {"status": "Could not find repo group with name 'Frontend Repos'", "repo_url": url}
+
+            repo_group_id = frontend_repo_group.repo_group_id
+
+
+        repo_id = Repo.insert_gitlab_repo(session, url, repo_group_id, "Frontend")
+        if not repo_id:
+            return False, {"status": "Repo insertion failed", "repo_url": url}
+
+        result = UserRepo.insert(session, repo_id, group_id)
+        if not result:
+            return False, {"status": "repo_user insertion failed", "repo_url": url}
+
+        #collection_status records are now only added during collection -IM 5/1/23
+        #status = CollectionStatus.insert(session, repo_id)
+        #if not status:
+        #    return False, {"status": "Failed to create status for repo", "repo_url": url}
+
+        return True, {"status": "Repo Added", "repo_url": url}
 
     @staticmethod
-    def add(session, url: List[str], user_id: int, group_name=None, group_id=None, from_org_list=False, repo_type=None, repo_group_id=None) -> dict:
+    def add_github_repo(session, url: List[str], user_id: int, group_name=None, group_id=None, from_org_list=False, repo_type=None, repo_group_id=None) -> dict:
         """Add repo to the user repo table
 
         Args:
@@ -820,7 +893,7 @@ def add(session, url: List[str], user_id: int, group_name=None, group_id=None, f
             repo_group_id = frontend_repo_group.repo_group_id
 
 
-        repo_id = Repo.insert(session, url, repo_group_id, "Frontend", repo_type)
+        repo_id = Repo.insert_github_repo(session, url, repo_group_id, "Frontend", repo_type)
         if not repo_id:
             return False, {"status": "Repo insertion failed", "repo_url": url}
 
@@ -862,7 +935,7 @@ def delete(session, repo_id:int, user_id:int, group_name:str) -> dict:
         return True, {"status": "Repo Removed"}
 
     @staticmethod
-    def add_org_repos(session, url: List[str], user_id: int, group_name: int):
+    def add_github_org_repos(session, url: List[str], user_id: int, group_name: int):
         """Add list of orgs and their repos to a users repos.
 
         Args:
@@ -911,7 +984,7 @@ def add_org_repos(session, url: List[str], user_id: int, group_name: int):
         failed_repos = []
         for repo in repos:
 
-            result = UserRepo.add(session, repo, user_id, group_id=group_id, from_org_list=True, repo_type=type, repo_group_id=repo_group_id)
+            result = UserRepo.add_github_repo(session, repo, user_id, group_id=group_id, from_org_list=True, repo_type=type, repo_group_id=repo_group_id)
 
             # keep track of all the repos that failed
             if not result[0]: