From be392cee00deed5f9c9a2d215e6c34ec4d9033c3 Mon Sep 17 00:00:00 2001 From: erew123 <35898566+erew123@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:56:08 +0100 Subject: [PATCH] Added compare You can now compare a base standalone installation with your diagnostic file and match versions. --- diagnostics.py | 553 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 539 insertions(+), 14 deletions(-) diff --git a/diagnostics.py b/diagnostics.py index 929744c..76819b5 100644 --- a/diagnostics.py +++ b/diagnostics.py @@ -9,7 +9,11 @@ import glob import site import textwrap import threading +import json +import traceback +import signal from pathlib import Path +from datetime import datetime # Check and install psutil if necessary try: @@ -37,6 +41,20 @@ except ImportError: from packaging.specifiers import SpecifierSet from packaging.version import parse as parse_version +try: + from PyQt6.QtWidgets import (QApplication, QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton, QLabel, + QFileDialog, QTableWidget, QTableWidgetItem, QHeaderView, QSplitter, QFrame) + from PyQt6.QtGui import QColor, QDropEvent, QDragEnterEvent, QFont + from PyQt6.QtCore import Qt, QTimer +except ImportError: + print("PyQt6 not found. Installing...") + subprocess.run([sys.executable, '-m', 'pip', 'install', 'PyQt6']) + from PyQt6.QtWidgets import (QApplication, QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton, QLabel, + QFileDialog, QTableWidget, QTableWidgetItem, QHeaderView, QSplitter, QFrame) + from PyQt6.QtGui import QColor, QDropEvent, QDragEnterEvent, QFont + from PyQt6.QtCore import Qt, QTimer + + def input_with_timeout(prompt, timeout): def input_thread(result): try: @@ -64,12 +82,6 @@ def get_requirements_file(): if not requirements_files: print("\033[91mNo requirements files found.\033[0m") return None - print(f"\033[94m _ _ _ \033[1;35m_____ _ _ \033[0m _____ _____ ____ ") - print(f"\033[94m / \ | | |\033[1;35m_ _|_ _| | | __ \033[0m |_ _|_ _/ ___| ") - print(f"\033[94m / _ \ | | |\033[1;35m | |/ _` | | |/ / \033[0m | | | | \___ \ ") - print(f"\033[94m / ___ \| | |\033[1;35m | | (_| | | < \033[0m | | | | ___) |") - print(f"\033[94m /_/ \_\_|_|\033[1;35m |_|\__,_|_|_|\_\ \033[0m |_| |_| |____/ ") - print(f"") print("\033[94m\n Select a requirements file to check against.") print("\033[94m Or press Enter for default 'requirements_standalone.txt'):\033[0m\n") @@ -86,10 +98,37 @@ def get_requirements_file(): except (ValueError, IndexError): return str(requirements_dir / "requirements_standalone.txt") -logging.basicConfig(filename='diagnostics.log', - filemode='w', - level=logging.INFO, - format='%(message)s') +def setup_logging(): + log_file = 'diagnostics.log' + + # Close existing handlers + for handler in logging.root.handlers[:]: + handler.close() + logging.root.removeHandler(handler) + + # Delete the existing log file if it exists + if os.path.exists(log_file): + try: + os.remove(log_file) + print(f" Existing {log_file} has been deleted.") + except Exception as e: + print(f" Error deleting existing {log_file}: {e}") + return # Exit the function if we can't delete the file + else: + print(f" Creating new {log_file} file.\n\n") + + # Set up logging + try: + logging.basicConfig(filename=log_file, + filemode='w', + level=logging.INFO, + format='%(message)s') + + logging.info(f"Diagnostic log created at {datetime.now()}") + logging.info("="*50 + "\n") + print(f" New {log_file} has been set up successfully.\n\n") + except Exception as e: + print(f" Error setting up logging: {e}") def get_gpu_info(): try: @@ -190,11 +229,75 @@ def find_files_in_path_with_wildcard(pattern): found_paths.append(file_path) return found_paths +def get_conda_info(): + conda_exe = os.environ.get('CONDA_EXE') + if not conda_exe: + return "CONDA_EXE environment variable not found. Conda might not be installed or properly set up." + + conda_info = {} + try: + # Get general conda info + result = subprocess.run([conda_exe, 'info', '--json'], capture_output=True, text=True) + if result.returncode == 0: + conda_info = json.loads(result.stdout) + else: + return f"Error running conda info: {result.stderr}" + + # Get list of packages in current environment + result = subprocess.run([conda_exe, 'list', '--json'], capture_output=True, text=True) + if result.returncode == 0: + packages = json.loads(result.stdout) + else: + return f"Error getting package list: {result.stderr}" + + except Exception as e: + return f"Error executing conda: {str(e)}" + + # Extract relevant information + conda_version = conda_info.get('conda_version', 'Unknown') + current_env = conda_info.get('active_prefix', 'No active environment') + env_list = conda_info.get('envs', []) + + # Process package information + package_info = { + 'conda-forge': {}, + 'pkgs/main': {}, + 'pkgs/msys2': {}, + 'other': {} + } + for package in packages: + name = package.get('name', 'Unknown') + version = package.get('version', 'Unknown') + channel = package.get('channel', 'Unknown') + + # Skip pypi (pip-installed) packages + if channel == 'pypi': + continue + + # Categorize packages + if 'conda-forge' in channel: + package_info['conda-forge'][name] = {'version': version, 'channel': channel} + elif channel == 'pkgs/main': + package_info['pkgs/main'][name] = {'version': version, 'channel': channel} + elif channel == 'pkgs/msys2': + package_info['pkgs/msys2'][name] = {'version': version, 'channel': channel} + elif 'nvidia' not in channel.lower(): # Exclude NVIDIA packages + package_info['other'][name] = {'version': version, 'channel': channel} + + return { + 'version': conda_version, + 'current_env': current_env, + 'env_list': env_list, + 'conda_exe': conda_exe, + 'packages': package_info + } + def log_system_info(): os_version = platform.system() + " " + platform.version() cuda_home = os.environ.get('CUDA_HOME', 'N/A') gpu_info = get_gpu_info() python_version = platform.python_version() + conda_info = get_conda_info() torch_version = torch.__version__ cuda_test_result = test_cuda() cuda_device_name, cuda_device_memory, cuda_version = get_cuda_details() @@ -285,7 +388,24 @@ def log_system_info(): logging.info("\nOS SEARCHPATH ENVIRONMENT:") for path in path_env.split(';'): logging.info(f" {path}") - + logging.info("\nCONDA INFORMATION:") + if isinstance(conda_info, str): + logging.info(f" {conda_info}") + else: + logging.info(f" Conda Executable: {conda_info['conda_exe']}") + logging.info(f" Conda Version: {conda_info['version']}") + logging.info(f" Current Environment: {conda_info['current_env']}") + logging.info(" Conda Environments:") + for env in conda_info['env_list']: + logging.info(f" - {env}") + + logging.info("\nCONDA PACKAGES IN CURRENT ENVIRONMENT:") + for category in ['conda-forge', 'pkgs/main', 'pkgs/msys2', 'other']: + if conda_info['packages'][category]: + logging.info(f"\n {category.upper()} PACKAGES:") + for package, details in conda_info['packages'][category].items(): + logging.info(f" {package:<30} Version: {details['version']:<15} Channel: {details['channel']}") + if required_packages: logging.info("\nPACKAGE VERSIONS vs REQUIREMENTS FILE:") max_package_length = max(len(package) for package in required_packages.keys()) @@ -294,8 +414,10 @@ def log_system_info(): logging.info(f" {package_name.ljust(max_package_length)} Required: {operator} {required_version.ljust(12)} Installed: {installed_version}") logging.info("\nPYTHON PACKAGES:") + package_versions = {d.metadata['Name']: d.version for d in distributions()} + max_package_length = max(len(package) for package in package_versions.keys()) for package, version in package_versions.items(): - logging.info(f" {package}>= {version}") + logging.info(f" {package:<{max_package_length}} = {version}") deepspeed_requirements = ( f"\nDeepSpeed Installation Requirements:\n" @@ -350,6 +472,38 @@ def log_system_info(): print(f"\033[0m 'text-generation-webui' listed in the path of the above folders. If you dont") print(f"\033[0m see them mentioned, you have probably not started the correct Python virtual") print(f"\033[0m environment.") + print(f"\n\033[94mConda Information:\033[0m") + if isinstance(conda_info, str): + print(f"\033[92m{conda_info}\033[0m") + else: + print(f"\033[94mConda Executable:\033[0m \033[92m{conda_info['conda_exe']}\033[0m") + print(f"\033[94mConda Version:\033[0m \033[92m{conda_info['version']}\033[0m") + print(f"\033[94mCurrent Environment:\033[0m \033[92m{conda_info['current_env']}\033[0m") + + # Check for faiss-cpu in conda-forge and pytorch, and ffmpeg in conda-forge + print(f"\n\033[94mKey Conda Packages:\033[0m") + all_conda_packages = { + **conda_info['packages']['conda-forge'], + **conda_info['packages']['other'] + } + + # Check for faiss-cpu + if 'faiss-cpu' in all_conda_packages: + details = all_conda_packages['faiss-cpu'] + channel = "conda-forge" if "conda-forge" in details['channel'] else "pytorch" if "pytorch" in details['channel'] else details['channel'] + print(f" \033[92mfaiss-cpu\033[0m Version: \033[93m{details['version']:<15}\033[0m Channel: \033[94m{channel}\033[0m") + else: + print(f" \033[91mfaiss-cpu\033[0m Not installed via conda") + + # Check for ffmpeg + if 'ffmpeg' in conda_info['packages']['conda-forge']: + details = conda_info['packages']['conda-forge']['ffmpeg'] + print(f" \033[92mffmpeg \033[0m Version: \033[93m{details['version']:<15}\033[0m Channel: \033[94mconda-forge\033[0m") + else: + print(f" \033[91mffmpeg \033[0m Not installed via conda-forge") + + + if required_packages: print("\033[94m\nRequirements file package comparison:\033[0m") max_package_length = max(len(package) for package in required_packages.keys()) @@ -398,6 +552,377 @@ def log_system_info(): print(f"\033[94mtime to check them. \033[0m") print(f"\033[94m\nIf you are going to ask for support, please upload the \033[92mdiagnostics.log\033[94m file.\033[0m") + + +class DragDropTextEdit(QTextEdit): + def __init__(self, parent=None): + super().__init__(parent) + self.setAcceptDrops(True) + + def dragEnterEvent(self, event: QDragEnterEvent): + if event.mimeData().hasUrls(): + event.acceptProposedAction() + + def dropEvent(self, event: QDropEvent): + for url in event.mimeData().urls(): + file_path = url.toLocalFile() + with open(file_path, 'r') as file: + self.setText(file.read()) + +class PackageComparisonTool(QWidget): + def __init__(self): + super().__init__() + try: + self.initUI() + except Exception as e: + print(f"Error in PackageComparisonTool initialization: {str(e)}") + traceback.print_exc() + + def __init__(self): + super().__init__() + self.initUI() + + def initUI(self): + main_layout = QVBoxLayout() # Main layout is vertical + + # Set default font and styles for the entire widget + self.setStyleSheet(""" + QWidget { + font-size: 10pt; + } + QLabel { + font-weight: bold; + } + QPushButton { + font-size: 10pt; + font-weight: bold; + padding: 5px; + } + QPushButton[accessibleName="browse"] { + font-weight: normal; + } + """) + + # Instructions + instructions = QLabel( + "INSTRUCTIONS:

" + "    1. Base Diagnostics File: Load the base diagnostics log file (e.g. from a working setup). Try `/system/config/basediagnostics.log`
" + "    2. Comparison Diagnostics File: Load your current diagnostics log file
" + "    3. Click the 'Compare' button to see differences
" + "    4. Review results in the table and organized results
" + "    5. Use 'Copy Commands' to copy pip commands to manually use at the Python command prompt.
" + "    6. Optionally, use 'Run Pip Commands' to apply changes (requires AllTalk's Python environment is activated).
" + "        You can watch the installation at the terminal/command prompt.
" + ) + instructions.setWordWrap(True) + main_layout.addWidget(instructions) + + # File inputs + file_layout = QHBoxLayout() + + # Base file input + base_layout = QVBoxLayout() + base_label = QLabel("Base Diagnostics File (Drag&Drop or Browse):") + self.base_text = DragDropTextEdit() + base_browse = QPushButton("Browse for a Base Diagnostic log file") + base_browse.setAccessibleName("browse") + base_browse.clicked.connect(lambda: self.browse_file(self.base_text)) + base_layout.addWidget(base_label) + base_layout.addWidget(self.base_text) + base_layout.addWidget(base_browse) + + # Compare file input + compare_layout = QVBoxLayout() + compare_label = QLabel("Comparison Diagnostics File (Drag&Drop or Browse):") + self.compare_text = DragDropTextEdit() + compare_browse = QPushButton("Browse for your Comparison Diagnostic log file") + compare_browse.setAccessibleName("browse") + compare_browse.clicked.connect(lambda: self.browse_file(self.compare_text)) + compare_layout.addWidget(compare_label) + compare_layout.addWidget(self.compare_text) + compare_layout.addWidget(compare_browse) + + file_layout.addLayout(base_layout) + file_layout.addLayout(compare_layout) + + main_layout.addLayout(file_layout) + + # Compare button + compare_button = QPushButton("Compare the Base and Comparision log files") + compare_button.clicked.connect(self.compare_packages) + + main_layout.addWidget(compare_button) + main_layout.addSpacing(20) + + # Results table + self.result_table = QTableWidget() + self.result_table.setColumnCount(3) + self.result_table.setHorizontalHeaderLabels(["Package", "Base Version", "Comparison Version"]) + self.result_table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch) + main_layout.addWidget(self.result_table) + main_layout.addSpacing(20) + + lower_section_layout = QHBoxLayout() + + # Pip commands box + pip_layout = QVBoxLayout() + pip_label = QLabel("Pip commands to align versions:") + pip_layout.addWidget(pip_label) + + self.pip_commands = QTextEdit() + self.pip_commands.setReadOnly(True) + pip_layout.addWidget(self.pip_commands) + + pip_buttons_layout = QHBoxLayout() + copy_button = QPushButton("Copy Commands") + copy_button.clicked.connect(self.copy_pip_commands) + run_pip_button = QPushButton("Run Pip Commands") + run_pip_button.clicked.connect(self.run_pip_commands) + pip_buttons_layout.addWidget(copy_button) + pip_buttons_layout.addWidget(run_pip_button) + + pip_layout.addLayout(pip_buttons_layout) + lower_section_layout.addLayout(pip_layout) + + # Add some spacing between the two sections + lower_section_layout.addSpacing(20) + + # Organized results box + org_layout = QVBoxLayout() + org_label = QLabel("Organized Results:") + org_layout.addWidget(org_label) + + self.org_results = QTextEdit() + self.org_results.setReadOnly(True) + org_layout.addWidget(self.org_results) + + copy_org_button = QPushButton("Copy Organized Results") + copy_org_button.clicked.connect(self.copy_organized_results) + org_layout.addWidget(copy_org_button) + + lower_section_layout.addLayout(org_layout) + + # Add the lower section layout to the main layout + main_layout.addLayout(lower_section_layout) + + self.setLayout(main_layout) + self.setGeometry(100, 100, 1200, 800) + self.setWindowTitle('Advanced Package Comparison Tool') + self.show() + + def browse_file(self, text_edit): + file_name, _ = QFileDialog.getOpenFileName(self, "Open File", "", "Text Files (*.txt);;Log Files (*.log);;All Files (*)") + if file_name: + try: + # Check if file exists and is not empty + if not os.path.exists(file_name): + raise FileNotFoundError(f"File does not exist: {file_name}") + + file_size = os.path.getsize(file_name) + print(f"File size: {file_size} bytes") + + if file_size == 0: + print(f"Warning: File is empty: {file_name}") + return + + # Try different encoding options + encodings = ['utf-8', 'utf-8-sig', 'ascii', 'latin-1'] + content = "" + + for encoding in encodings: + try: + with open(file_name, 'r', encoding=encoding) as file: + content = file.read() + print(f"File opened successfully with {encoding} encoding: {file_name}") + break + except UnicodeDecodeError: + print(f"Failed to open with {encoding} encoding, trying next...") + + if not content: + raise ValueError("Unable to read file content with any encoding") + + text_edit.setText(content) + print(f"Content length: {len(content)} characters") + + # Print first few lines for debugging + print("First few lines of content:") + print("\n".join(content.split('\n')[:5])) + + except Exception as e: + print(f"Error opening file {file_name}: {str(e)}") + traceback.print_exc() + + def extract_packages(self, text): + packages = {} + try: + start_index = text.find("PYTHON PACKAGES:") + if start_index != -1: + package_section = text[start_index:] + lines = package_section.split('\n')[1:] # Skip the "PYTHON PACKAGES:" line + for line in lines: + if line.strip() == "": + break # Stop at the first empty line after packages + parts = line.split('=') + if len(parts) == 2: + package_name = parts[0].strip() + version = parts[1].strip() + packages[package_name.lower()] = version + + # print(f"Extracted {len(packages)} packages") + except Exception as e: + print(f"Error extracting packages: {str(e)}") + traceback.print_exc() + + return packages + + def compare_packages(self): + base_text = self.base_text.toPlainText() + compare_text = self.compare_text.toPlainText() + base_packages = self.extract_packages(base_text) + compare_packages = self.extract_packages(compare_text) + + self.result_table.setRowCount(0) + pip_commands = [] + missing = [] + different = [] + matching = [] + additional = [] + + all_packages = sorted(set(base_packages.keys()) | set(compare_packages.keys()), key=str.lower) + + for package_lower in all_packages: + row_position = self.result_table.rowCount() + self.result_table.insertRow(row_position) + + # Find the original capitalization + package = next((p for p in base_packages.keys() if p.lower() == package_lower), + next((p for p in compare_packages.keys() if p.lower() == package_lower), package_lower)) + + base_version = base_packages.get(package_lower, "N/A") + compare_version = compare_packages.get(package_lower, "N/A") + + self.result_table.setItem(row_position, 0, QTableWidgetItem(package)) + self.result_table.setItem(row_position, 1, QTableWidgetItem(base_version)) + self.result_table.setItem(row_position, 2, QTableWidgetItem(compare_version)) + + if base_version == compare_version: + for col in range(3): + self.result_table.item(row_position, col).setBackground(QColor("lightgreen")) + matching.append(f"{package}: {base_version}") + elif base_version == "N/A": + for col in range(3): + self.result_table.item(row_position, col).setBackground(QColor("lightblue")) + additional.append(f"{package}: {compare_version}") + elif compare_version == "N/A": + for col in range(3): + self.result_table.item(row_position, col).setBackground(QColor("pink")) + pip_commands.append(f"pip install {package}=={base_version}") + missing.append(f"{package}: {base_version}") + else: + for col in range(3): + self.result_table.item(row_position, col).setBackground(QColor("yellow")) + pip_commands.append(f"pip install {package}=={base_version}") + different.append(f"{package}: {base_version} (Base) vs {compare_version} (Compare)") + + self.pip_commands.setText("\n".join(pip_commands)) + + organized_results = "Missing in Comparison (present in Base):\n" + "\n".join(sorted(missing)) + "\n\n" + organized_results += "Different Versions:\n" + "\n".join(sorted(different)) + "\n\n" + organized_results += "Matching Versions:\n" + "\n".join(sorted(matching)) + "\n\n" + organized_results += "Additional in Comparison (not in Base):\n" + "\n".join(sorted(additional)) + self.org_results.setText(organized_results) + + def copy_pip_commands(self): + clipboard = QApplication.clipboard() + clipboard.setText(self.pip_commands.toPlainText()) + + def copy_organized_results(self): + clipboard = QApplication.clipboard() + clipboard.setText(self.org_results.toPlainText()) + + def run_pip_commands(self): + commands = self.pip_commands.toPlainText().split('\n') + for command in commands: + try: + subprocess.run(command, shell=True, check=True) + print(f"Successfully executed: {command}") + except subprocess.CalledProcessError as e: + print(f"Error executing {command}: {e}") + +def cleanup_logging(): + logging.shutdown() + +# Global flag to indicate if the application should exit +should_exit = False + +def signal_handler(signum, frame): + global should_exit + print("\nCtrl+C pressed. Closing the application.") + should_exit = True + +# Set up the signal handler at the global level +signal.signal(signal.SIGINT, signal_handler) + + +def show_menu(): + print("\n\n\n\033[94m AllTalk Diagnostics Tool Menu\033[0m") + print("\n 1. \033[93mGenerate a \033[92mdiagnostics.log\033[93m file\033[0m") + print(" This option creates a new detailed log file of your system's configuration,") + print(" including OS, hardware, Python environment, and installed packages. Use this") + print(" to troubleshoot issues or when seeking support. Old diagnostic.log files will") + print(" be overwritten.") + print("\n 2. \033[93mCompare two \033[92mdiagnostics.log\033[93m logs\033[0m") + print(" This opens a graphical tool to compare two diagnostics.log files. Useful") + print(" for identifying differences between working and non-working configurations") + print(" or for comparing system setups.") + print("\n 9. \033[91mExit\033[0m") + print(" Close the diagnostics tool.") + choice = input("\n Enter your choice (1, 2 or exit with 9): ") + return choice + +def main(): + print(f"\n\033[94m _ _ _ \033[1;35m_____ _ _ \033[0m _____ _____ ____ ") + print(f"\033[94m / \ | | |\033[1;35m_ _|_ _| | | __ \033[0m |_ _|_ _/ ___| ") + print(f"\033[94m / _ \ | | |\033[1;35m | |/ _` | | |/ / \033[0m | | | | \___ \ ") + print(f"\033[94m / ___ \| | |\033[1;35m | | (_| | | < \033[0m | | | | ___) |") + print(f"\033[94m /_/ \_\_|_|\033[1;35m |_|\__,_|_|_|\_\ \033[0m |_| |_| |____/ ") + + app = QApplication(sys.argv) + + # Use a timer to check for the exit flag + timer = QTimer() + timer.timeout.connect(lambda: check_exit(app)) + timer.start(100) # Check every 100ms + + while not should_exit: + choice = show_menu() + if choice == '1': + setup_logging() + log_system_info() + print("\n Diagnostics completed. Check the \033[92mdiagnostics.log\033[0m file for details.\n") + input(" Press Enter to return to the main menu...") + elif choice == '2': + try: + ex = PackageComparisonTool() + ex.show() + app.exec() + except Exception as e: + print(f"Error initializing PackageComparisonTool: {str(e)}") + traceback.print_exc() + elif choice == '9': + cleanup_logging() + print("\n Exiting...") + break + else: + print(" Invalid choice. Please try again.") + + cleanup_logging() + app.quit() + +def check_exit(app): + if should_exit: + cleanup_logging() + app.quit() + if __name__ == "__main__": - log_system_info() - sys.exit(1) + main() \ No newline at end of file