komehdaht/miscellaneous
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Ruslan Piatrovich
2024-08-07 16:52:02 +03:00
commit 9e9b1af438
1356 changed files with 210505 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
check_external_repos.py
+363
View File
@@ -0,0 +1,363 @@
import os
import time
import requests
from github import Github
import base64
import json
import re
import concurrent.futures
from retrying import retry
from functools import wraps
# Список токенов
TOKENS = [
'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',
'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',
'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',
'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'
# Добавьте столько токенов, сколько нужно
]
TOKEN_INDEX = 0
ORG_NAME = 'adeo'
GITHUB_ENTERPRISE_URL = 'https://github.lmru.tech' # URL вашей инстанции GitHub Enterprise
OUTPUT_FILE = 'external_repositories.txt'
CACHE_DIR = 'repo_cache'
REPOS_CACHE_FILE = 'repos_cache.json'
MAX_WORKERS = 16 # Начнем с 8 потоков
# Регулярное выражение для поиска URL
url_regex = re.compile(r'https?://[^\s]+')
excluded_url_pattern = re.compile(r'https?://art\.lmru\.tech')
# Уникальные внешние URL
unique_urls = set()
# Создание директории для кэша
if not os.path.exists(CACHE_DIR):
os.makedirs(CACHE_DIR)
# Декоратор для задержки между запросами
def rate_limited(func):
@wraps(func)
def wrapper(*args, **kwargs):
time.sleep(1) # Задержка в 1 секунду между запросами
return func(*args, **kwargs)
return wrapper
# Функция для получения текущего токена
def get_current_token():
global TOKEN_INDEX
token = TOKENS[TOKEN_INDEX % len(TOKENS)]
TOKEN_INDEX += 1
return token
# Создание экземпляра GitHub API с аутентификацией и кастомным URL
def create_github_instance():
token = get_current_token()
return Github(base_url=f"{GITHUB_ENTERPRISE_URL}/api/v3", login_or_token=token)
@rate_limited
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5)
def get_repo_contents(repo, path=""):
return repo.get_contents(path)
@rate_limited
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5)
def get_file_content(file_content):
return base64.b64decode(file_content.content).decode('utf-8')
@rate_limited
def check_rate_limit():
headers = {
"Authorization": f"token {get_current_token()}"
}
rate_limit_url = f"{GITHUB_ENTERPRISE_URL}/api/v3/rate_limit"
response = requests.get(rate_limit_url, headers=headers)
if response.status_code == 200:
limits = response.json()
core_limit = limits["resources"]["core"]["limit"]
core_remaining = limits["resources"]["core"]["remaining"]
core_reset = limits["resources"]["core"]["reset"]
print(f"Core limit: {core_limit}")
print(f"Core remaining: {core_remaining}")
print(f"Core reset time: {time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(core_reset))}")
return core_remaining
else:
print(f"Failed to retrieve rate limits: {response.status_code}, {response.text}")
return None
@rate_limited
def fetch_all_repos():
repos = []
g = create_github_instance()
try:
for repo in g.get_organization(ORG_NAME).get_repos():
repos.append(repo.full_name)
except Exception as e:
print(f'Error retrieving organization repositories: {str(e)}')
if '403' in str(e):
print('Hit rate limit, sleeping for 60 seconds...')
time.sleep(60)
print(f"Total repositories retrieved: {len(repos)}") # Логирование количества репозиториев
return repos
def get_all_repos():
if os.path.exists(REPOS_CACHE_FILE):
with open(REPOS_CACHE_FILE, 'r') as f:
repos = json.load(f)
print(f"Loaded {len(repos)} repositories from cache")
else:
repos = fetch_all_repos()
with open(REPOS_CACHE_FILE, 'w') as f:
json.dump(repos, f)
print(f"Saved {len(repos)} repositories to cache")
return repos
def cache_file_path(repo_name, file_path):
return os.path.join(CACHE_DIR, f"{repo_name.replace('/', '_')}_{file_path.replace('/', '_')}")
def is_cached(repo_name, file_path):
return os.path.exists(cache_file_path(repo_name, file_path))
def read_cache(repo_name, file_path):
with open(cache_file_path(repo_name, file_path), 'r') as f:
return f.read()
def write_cache(repo_name, file_path, content):
with open(cache_file_path(repo_name, file_path), 'w') as f:
f.write(content)
def find_external_urls(content, repo_name, file_path):
urls = url_regex.findall(content)
for url in urls:
if not excluded_url_pattern.search(url):
unique_urls.add(url)
print(f'Found external URL in {repo_name}/{file_path}: {url}')
def check_file(repo, file_content):
file_path = file_content.path
if is_cached(repo.name, file_path):
print(f'Reading from cache: {repo.name}/{file_path}')
content = read_cache(repo.name, file_path)
else:
try:
# Чтение содержимого файла с повторными попытками
content = get_file_content(file_content)
write_cache(repo.name, file_path, content)
except Exception as e:
print(f'Error decoding {file_path} in {repo.name}: {str(e)}')
return
# Поиск внешних URL
find_external_urls(content, repo.name, file_path)
def check_and_report_external_repositories(repo):
try:
contents = get_repo_contents(repo)
while contents:
file_content = contents.pop(0)
if file_content.type == "dir":
print(f'Entering directory: {file_content.path}')
contents.extend(get_repo_contents(repo, file_content.path))
else:
print(f'Checking file: {file_content.path}')
if file_content.path.endswith((
"package.json", "requirements.txt", "setup.py", "pom.xml", "build.gradle", "Cargo.toml", "composer.json",
"Makefile", "Dockerfile", "saltfile", "Pipfile", "environment.yml", "Jenkinsfile", "Gemfile", "Rakefile",
"go.mod", "go.sum", "build.sbt", "Vagrantfile", "shard.yml", "Chart.yaml", "mix.exs", "nimble", "rebar.config",
"Project.toml", "deps.edn", "shadow-cljs.edn", "Spago.dhall", "flake.nix", "default.nix", "pubspec.yaml",
"Paket.dependencies", "Packages.swift", "universe.json", "stack.yaml", "hpack.yaml", "default.build", "project.clj"
)):
check_file(repo, file_content)
except Exception as e:
if '404' in str(e):
print(f'Skipping empty repository {repo.name}')
else:
print(f'Error processing repository {repo.name}: {str(e)}')
def process_repository(repo_name):
print(f'Checking repository: {repo_name}')
g = create_github_instance()
repo = g.get_repo(repo_name)
check_and_report_external_repositories(repo)
print(f'Finished checking repository: {repo_name}')
start_time = time.time()
# Проверка лимитов перед началом выполнения
if check_rate_limit() is None:
exit(1)
# Получение всех репозиториев
repos = get_all_repos()
# Проверка, что репозитории были получены
if not repos:
print("No repositories retrieved. Exiting.")
exit(1)
# Использование пула потоков для обработки репозиториев
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
executor.map(process_repository, repos)
end_time = time.time()
elapsed_time = end_time - start_time
# Запись уникальных внешних URL в файл
with open(OUTPUT_FILE, 'w') as file:
for url in unique_urls:
file.write(f"{url}\n")
print(f'Unique external URLs have been written to {OUTPUT_FILE}')
print(f'Total time taken: {elapsed_time / 60:.2f} minutes')
# Проверка лимитов после выполнения
check_rate_limit()
import os
import subprocess
import time
import shutil
import re
from github import Github
import concurrent.futures
import json
# Список токенов
TOKENS = [
'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',
'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',
'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',
'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'
# Добавьте столько токенов, сколько нужно
]
TOKEN_INDEX = 0
ORG_NAME = 'adeo'
GITHUB_ENTERPRISE_URL = 'https://github.lmru.tech'
CLONE_DIR = '/tmp/github_test' # Укажите путь к директории, куда будут клонироваться репозитории
REPOS_CACHE_FILE = 'repos_cache.json'
OUTPUT_FILE = 'external_repositories.txt'
MAX_WORKERS = 8 # Количество параллельных процессов для клонирования
BATCH_SIZE = 50 # Количество репозиториев в одной партии для обработки
# Регулярное выражение для поиска URL
url_regex = re.compile(r'https?://[^\s]+')
excluded_url_pattern = re.compile(r'https?://art\.lmru\.tech')
# Уникальные внешние URL
unique_urls = set()
# Функция для получения текущего токена
def get_current_token():
global TOKEN_INDEX
token = TOKENS[TOKEN_INDEX % len(TOKENS)]
TOKEN_INDEX += 1
return token
# Создание экземпляра GitHub API с аутентификацией и кастомным URL
def create_github_instance():
token = get_current_token()
return Github(base_url=f"{GITHUB_ENTERPRISE_URL}/api/v3", login_or_token=token)
# Получение всех репозиториев и сохранение их в файл кэша
def fetch_all_repos():
repos = []
g = create_github_instance()
try:
for repo in g.get_organization(ORG_NAME).get_repos():
repos.append(repo.clone_url)
except Exception as e:
print(f'Error retrieving organization repositories: {str(e)}')
if '403' in str(e):
print('Hit rate limit, sleeping for 60 seconds...')
time.sleep(60)
print(f"Total repositories retrieved: {len(repos)}")
with open(REPOS_CACHE_FILE, 'w') as f:
json.dump(repos, f)
return repos
# Получение списка репозиториев из кэша
def get_all_repos():
if os.path.exists(REPOS_CACHE_FILE):
with open(REPOS_CACHE_FILE, 'r') as f:
repos = json.load(f)
print(f"Loaded {len(repos)} repositories from cache")
else:
repos = fetch_all_repos()
return repos
# Клонирование репозитория
def clone_repo(repo_url):
try:
token = get_current_token()
repo_name = repo_url.split('/')[-1].replace('.git', '')
clone_url = repo_url.replace('https://', f'https://{token}:x-oauth-basic@')
clone_path = os.path.join(CLONE_DIR, repo_name)
if not os.path.exists(clone_path):
subprocess.run(['git', 'clone', clone_url, clone_path])
print(f'Cloned {repo_url} to {clone_path}')
else:
print(f'Repository {repo_name} already exists, pulling latest changes')
subprocess.run(['git', '-C', clone_path, 'pull'])
except Exception as e:
print(f'Error cloning repository {repo_url}: {str(e)}')
# Поиск внешних URL в файлах
def find_external_urls(file_path):
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
urls = url_regex.findall(content)
for url in urls:
if not excluded_url_pattern.search(url):
unique_urls.add(url)
print(f'Found external URL in {file_path}: {url}')
# Анализ репозитория
def analyze_repo(repo_path):
for root, _, files in os.walk(repo_path):
for file in files:
if file.endswith((
"package.json", "requirements.txt", "setup.py", "pom.xml", "build.gradle", "Cargo.toml", "composer.json",
"Makefile", "Dockerfile", "saltfile", "Pipfile", "environment.yml", "Jenkinsfile", "Gemfile", "Rakefile",
"go.mod", "go.sum", "build.sbt", "Vagrantfile", "shard.yml", "Chart.yaml", "mix.exs", "nimble", "rebar.config",
"Project.toml", "deps.edn", "shadow-cljs.edn", "Spago.dhall", "flake.nix", "default.nix", "pubspec.yaml",
"Paket.dependencies", "Packages.swift", "universe.json", "stack.yaml", "hpack.yaml", "default.build", "project.clj"
)):
find_external_urls(os.path.join(root, file))
# Обработка репозиториев партиями
def process_repos_in_batches(repos):
for i in range(0, len(repos), BATCH_SIZE):
batch = repos[i:i + BATCH_SIZE]
# Клонирование репозиториев
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
executor.map(clone_repo, batch)
# Анализ клонированных репозиториев
for repo_url in batch:
repo_name = repo_url.split('/')[-1].replace('.git', '')
repo_path = os.path.join(CLONE_DIR, repo_name)
if os.path.isdir(repo_path):
analyze_repo(repo_path)
# Очистка после анализа
for repo_url in batch:
repo_name = repo_url.split('/')[-1].replace('.git', '')
repo_path = os.path.join(CLONE_DIR, repo_name)
if os.path.isdir(repo_path):
shutil.rmtree(repo_path)
print(f'Removed {repo_path} after analysis')
# Основной код
if __name__ == '__main__':
repos = get_all_repos()
process_repos_in_batches(repos)
# Запись уникальных внешних URL в файл
with open(OUTPUT_FILE, 'w') as file:
for url in unique_urls:
file.write(f"{url}\n")
print(f'Unique external URLs have been written to {OUTPUT_FILE}')