0n1-oss/hulud_check
3
0
Fork 0
Code Pull Requests Activity

Fix pagination, add group filtering option

Browse Source
This commit is contained in:
Adrian Marquis
2025-11-26 13:36:47 +01:00
parent 30e3d52153
commit ba550f8dc5
1 changed files with 22 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
check_gitlab.py
View File

@@ -12,12 +12,11 @@
# apt-get install ripgrep # apt-get install ripgrep
import os import os
import re
import subprocess import subprocess
import json import json
import csv import csv
from git import Repo from git import Repo
from requests import Request, Session from requests import Session
from pathlib import Path from pathlib import Path
class GitlabConnector: class GitlabConnector:
@@ -54,22 +53,34 @@ class Report():
for row in self.findings: for row in self.findings:
writer.writerow(row) writer.writerow(row)
def get_all_projects(next_link=None, prev_result=[]): def get_all_projects(next_link=None, group_id=None, prev_result=[]):
base_path = '/api/v4'
url_params = ["include_subgroups=true", "per_page=50", "search_namespaces=true", "owned=false", "order_by=id", "sort=asc"]
if group_id:
base_path += f"/groups/{group_id}"
else:
url_params.append("pagination=keyset")
if not next_link: if not next_link:
result = session.query("/api/v4/projects?pagination=keyset&per_page=50&search_namespaces=true&owned=false&order_by=id&sort=asc") result = session.query(f"{base_path}/projects?{'&'.join(url_params)}")
else: else:
result = session.get(next_link) result = session.get(next_link)
if result.headers.get('Link'): if result.headers.get('Link'):
link = result.headers['Link'].split(';')[0].replace('<', '').replace('>', '') links = result.headers['Link'].split(', ')
rel = result.headers['Link'].split(';')[1].split('=')[1] for link in links:
parts = link.split('; ')
rel = parts[1].split('=')[1]
if rel == '"next"':
link = parts[0].replace('<', '').replace('>', '')
break
prev_result += [{'id': i['id'], 'http_url_to_repo': i['http_url_to_repo'], 'ssh_url_to_repo': i['ssh_url_to_repo'], 'web_url': i['web_url']} for i in result.json()] prev_result += [{'id': i['id'], 'http_url_to_repo': i['http_url_to_repo'], 'ssh_url_to_repo': i['ssh_url_to_repo'], 'web_url': i['web_url']} for i in result.json()]
# I know, not nice.. but im in a hurry # I know, not nice.. but im in a hurry
try: try:
if rel == "\"next\"": if rel == "\"next\"":
get_all_projects(next_link=link, prev_result=prev_result) get_all_projects(next_link=link, group_id=group_id, prev_result=prev_result)
except: except:
pass pass
return prev_result return prev_result
@@ -146,7 +157,7 @@ def check_line_in_file(file=None, line_number=None):
def check_repos(): def check_repos():
repos = get_all_projects() repos = get_all_projects(group_id=os.environ.get('GITLAB_GROUP'))
print(f"Found {len(repos)} Repositories..") print(f"Found {len(repos)} Repositories..")
for repo in repos: for repo in repos:
scan_path = clone_repo_with_http(repo['http_url_to_repo']) scan_path = clone_repo_with_http(repo['http_url_to_repo'])
@@ -167,4 +178,4 @@ session = GitlabConnector()
report = Report() report = Report()
check_repos() check_repos()
report.results() report.results()
report.write_report(f"{report_path}/{report_file}") report.write_report(f"{report_path}/{report_file}")