Added compare

You can now compare a base standalone installation with your diagnostic file and match versions.
This commit is contained in:
erew123 authored and GitHub committed 2024-09-27 21:56:08 +01:00
1 parent 18b1d49b72
commit be392cee00
1 file changed
+539 -14
+539 -14
View File
@@ -9,7 +9,11 @@ import glob
import site
import textwrap
import threading
import json
import traceback
import signal
from pathlib import Path
from datetime import datetime
# Check and install psutil if necessary
try:
@@ -37,6 +41,20 @@ except ImportError:
from packaging.specifiers import SpecifierSet
from packaging.version import parse as parse_version
try:
from PyQt6.QtWidgets import (QApplication, QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton, QLabel,
QFileDialog, QTableWidget, QTableWidgetItem, QHeaderView, QSplitter, QFrame)
from PyQt6.QtGui import QColor, QDropEvent, QDragEnterEvent, QFont
from PyQt6.QtCore import Qt, QTimer
except ImportError:
print("PyQt6 not found. Installing...")
subprocess.run([sys.executable, '-m', 'pip', 'install', 'PyQt6'])
from PyQt6.QtWidgets import (QApplication, QWidget, QVBoxLayout, QHBoxLayout, QTextEdit, QPushButton, QLabel,
QFileDialog, QTableWidget, QTableWidgetItem, QHeaderView, QSplitter, QFrame)
from PyQt6.QtGui import QColor, QDropEvent, QDragEnterEvent, QFont
from PyQt6.QtCore import Qt, QTimer
def input_with_timeout(prompt, timeout):
def input_thread(result):
try:
@@ -64,12 +82,6 @@ def get_requirements_file():
if not requirements_files:
print("\033[91mNo requirements files found.\033[0m")
return None
print(f"\033[94m _ _ _ \033[1;35m_____ _ _ \033[0m _____ _____ ____ ")
print(f"\033[94m / \ | | |\033[1;35m_ _|_ _| | | __ \033[0m |_ _|_ _/ ___| ")
print(f"\033[94m / _ \ | | |\033[1;35m | |/ _` | | |/ / \033[0m | | | | \___ \ ")
print(f"\033[94m / ___ \| | |\033[1;35m | | (_| | | < \033[0m | | | | ___) |")
print(f"\033[94m /_/ \_\_|_|\033[1;35m |_|\__,_|_|_|\_\ \033[0m |_| |_| |____/ ")
print(f"")
print("\033[94m\n Select a requirements file to check against.")
print("\033[94m Or press Enter for default 'requirements_standalone.txt'):\033[0m\n")
@@ -86,10 +98,37 @@ def get_requirements_file():
except (ValueError, IndexError):
return str(requirements_dir / "requirements_standalone.txt")
logging.basicConfig(filename='diagnostics.log',
filemode='w',
level=logging.INFO,
format='%(message)s')
def setup_logging():
log_file = 'diagnostics.log'
# Close existing handlers
for handler in logging.root.handlers[:]:
handler.close()
logging.root.removeHandler(handler)
# Delete the existing log file if it exists
if os.path.exists(log_file):
try:
os.remove(log_file)
print(f" Existing {log_file} has been deleted.")
except Exception as e:
print(f" Error deleting existing {log_file}: {e}")
return # Exit the function if we can't delete the file
else:
print(f" Creating new {log_file} file.\n\n")
# Set up logging
try:
logging.basicConfig(filename=log_file,
filemode='w',
level=logging.INFO,
format='%(message)s')
logging.info(f"Diagnostic log created at {datetime.now()}")
logging.info("="*50 + "\n")
print(f" New {log_file} has been set up successfully.\n\n")
except Exception as e:
print(f" Error setting up logging: {e}")
def get_gpu_info():
try:
@@ -190,11 +229,75 @@ def find_files_in_path_with_wildcard(pattern):
found_paths.append(file_path)
return found_paths
def get_conda_info():
conda_exe = os.environ.get('CONDA_EXE')
if not conda_exe:
return "CONDA_EXE environment variable not found. Conda might not be installed or properly set up."
conda_info = {}
try:
# Get general conda info
result = subprocess.run([conda_exe, 'info', '--json'], capture_output=True, text=True)
if result.returncode == 0:
conda_info = json.loads(result.stdout)
else:
return f"Error running conda info: {result.stderr}"
# Get list of packages in current environment
result = subprocess.run([conda_exe, 'list', '--json'], capture_output=True, text=True)
if result.returncode == 0:
packages = json.loads(result.stdout)
else:
return f"Error getting package list: {result.stderr}"
except Exception as e:
return f"Error executing conda: {str(e)}"
# Extract relevant information
conda_version = conda_info.get('conda_version', 'Unknown')
current_env = conda_info.get('active_prefix', 'No active environment')
env_list = conda_info.get('envs', [])
# Process package information
package_info = {
'conda-forge': {},
'pkgs/main': {},
'pkgs/msys2': {},
'other': {}
}
for package in packages:
name = package.get('name', 'Unknown')
version = package.get('version', 'Unknown')
channel = package.get('channel', 'Unknown')
# Skip pypi (pip-installed) packages
if channel == 'pypi':
continue
# Categorize packages
if 'conda-forge' in channel:
package_info['conda-forge'][name] = {'version': version, 'channel': channel}
elif channel == 'pkgs/main':
package_info['pkgs/main'][name] = {'version': version, 'channel': channel}
elif channel == 'pkgs/msys2':
package_info['pkgs/msys2'][name] = {'version': version, 'channel': channel}
elif 'nvidia' not in channel.lower(): # Exclude NVIDIA packages
package_info['other'][name] = {'version': version, 'channel': channel}
return {
'version': conda_version,
'current_env': current_env,
'env_list': env_list,
'conda_exe': conda_exe,
'packages': package_info
}
def log_system_info():
os_version = platform.system() + " " + platform.version()
cuda_home = os.environ.get('CUDA_HOME', 'N/A')
gpu_info = get_gpu_info()
python_version = platform.python_version()
conda_info = get_conda_info()
torch_version = torch.__version__
cuda_test_result = test_cuda()
cuda_device_name, cuda_device_memory, cuda_version = get_cuda_details()
@@ -285,7 +388,24 @@ def log_system_info():
logging.info("\nOS SEARCHPATH ENVIRONMENT:")
for path in path_env.split(';'):
logging.info(f" {path}")
logging.info("\nCONDA INFORMATION:")
if isinstance(conda_info, str):
logging.info(f" {conda_info}")
else:
logging.info(f" Conda Executable: {conda_info['conda_exe']}")
logging.info(f" Conda Version: {conda_info['version']}")
logging.info(f" Current Environment: {conda_info['current_env']}")
logging.info(" Conda Environments:")
for env in conda_info['env_list']:
logging.info(f" - {env}")
logging.info("\nCONDA PACKAGES IN CURRENT ENVIRONMENT:")
for category in ['conda-forge', 'pkgs/main', 'pkgs/msys2', 'other']:
if conda_info['packages'][category]:
logging.info(f"\n {category.upper()} PACKAGES:")
for package, details in conda_info['packages'][category].items():
logging.info(f" {package:<30} Version: {details['version']:<15} Channel: {details['channel']}")
if required_packages:
logging.info("\nPACKAGE VERSIONS vs REQUIREMENTS FILE:")
max_package_length = max(len(package) for package in required_packages.keys())
@@ -294,8 +414,10 @@ def log_system_info():
logging.info(f" {package_name.ljust(max_package_length)} Required: {operator} {required_version.ljust(12)} Installed: {installed_version}")
logging.info("\nPYTHON PACKAGES:")
package_versions = {d.metadata['Name']: d.version for d in distributions()}
max_package_length = max(len(package) for package in package_versions.keys())
for package, version in package_versions.items():
logging.info(f" {package}>= {version}")
logging.info(f" {package:<{max_package_length}} = {version}")
deepspeed_requirements = (
f"\nDeepSpeed Installation Requirements:\n"
@@ -350,6 +472,38 @@ def log_system_info():
print(f"\033[0m 'text-generation-webui' listed in the path of the above folders. If you dont")
print(f"\033[0m see them mentioned, you have probably not started the correct Python virtual")
print(f"\033[0m environment.")
print(f"\n\033[94mConda Information:\033[0m")
if isinstance(conda_info, str):
print(f"\033[92m{conda_info}\033[0m")
else:
print(f"\033[94mConda Executable:\033[0m \033[92m{conda_info['conda_exe']}\033[0m")
print(f"\033[94mConda Version:\033[0m \033[92m{conda_info['version']}\033[0m")
print(f"\033[94mCurrent Environment:\033[0m \033[92m{conda_info['current_env']}\033[0m")
# Check for faiss-cpu in conda-forge and pytorch, and ffmpeg in conda-forge
print(f"\n\033[94mKey Conda Packages:\033[0m")
all_conda_packages = {
**conda_info['packages']['conda-forge'],
**conda_info['packages']['other']
}
# Check for faiss-cpu
if 'faiss-cpu' in all_conda_packages:
details = all_conda_packages['faiss-cpu']
channel = "conda-forge" if "conda-forge" in details['channel'] else "pytorch" if "pytorch" in details['channel'] else details['channel']
print(f" \033[92mfaiss-cpu\033[0m Version: \033[93m{details['version']:<15}\033[0m Channel: \033[94m{channel}\033[0m")
else:
print(f" \033[91mfaiss-cpu\033[0m Not installed via conda")
# Check for ffmpeg
if 'ffmpeg' in conda_info['packages']['conda-forge']:
details = conda_info['packages']['conda-forge']['ffmpeg']
print(f" \033[92mffmpeg \033[0m Version: \033[93m{details['version']:<15}\033[0m Channel: \033[94mconda-forge\033[0m")
else:
print(f" \033[91mffmpeg \033[0m Not installed via conda-forge")
if required_packages:
print("\033[94m\nRequirements file package comparison:\033[0m")
max_package_length = max(len(package) for package in required_packages.keys())
@@ -398,6 +552,377 @@ def log_system_info():
print(f"\033[94mtime to check them. \033[0m")
print(f"\033[94m\nIf you are going to ask for support, please upload the \033[92mdiagnostics.log\033[94m file.\033[0m")
class DragDropTextEdit(QTextEdit):
def __init__(self, parent=None):
super().__init__(parent)
self.setAcceptDrops(True)
def dragEnterEvent(self, event: QDragEnterEvent):
if event.mimeData().hasUrls():
event.acceptProposedAction()
def dropEvent(self, event: QDropEvent):
for url in event.mimeData().urls():
file_path = url.toLocalFile()
with open(file_path, 'r') as file:
self.setText(file.read())
class PackageComparisonTool(QWidget):
def __init__(self):
super().__init__()
try:
self.initUI()
except Exception as e:
print(f"Error in PackageComparisonTool initialization: {str(e)}")
traceback.print_exc()
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
main_layout = QVBoxLayout() # Main layout is vertical
# Set default font and styles for the entire widget
self.setStyleSheet("""
QWidget {
font-size: 10pt;
}
QLabel {
font-weight: bold;
}
QPushButton {
font-size: 10pt;
font-weight: bold;
padding: 5px;
}
QPushButton[accessibleName="browse"] {
font-weight: normal;
}
""")
# Instructions
instructions = QLabel(
"<b>INSTRUCTIONS:</b><br><br>"
"&nbsp;&nbsp;&nbsp;&nbsp;1. <b>Base Diagnostics File:</b> Load the base diagnostics log file (e.g. from a working setup). Try `/system/config/basediagnostics.log`<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;2. <b>Comparison Diagnostics File:</b> Load your current diagnostics log file<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;3. Click the 'Compare' button to see differences<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;4. Review results in the table and organized results<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;5. Use 'Copy Commands' to copy pip commands to manually use at the Python command prompt.<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;6. Optionally, use 'Run Pip Commands' to apply changes (requires AllTalk's Python environment is activated).<br>"
"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;You can watch the installation at the terminal/command prompt.<br>"
)
instructions.setWordWrap(True)
main_layout.addWidget(instructions)
# File inputs
file_layout = QHBoxLayout()
# Base file input
base_layout = QVBoxLayout()
base_label = QLabel("Base Diagnostics File (Drag&Drop or Browse):")
self.base_text = DragDropTextEdit()
base_browse = QPushButton("Browse for a Base Diagnostic log file")
base_browse.setAccessibleName("browse")
base_browse.clicked.connect(lambda: self.browse_file(self.base_text))
base_layout.addWidget(base_label)
base_layout.addWidget(self.base_text)
base_layout.addWidget(base_browse)
# Compare file input
compare_layout = QVBoxLayout()
compare_label = QLabel("Comparison Diagnostics File (Drag&Drop or Browse):")
self.compare_text = DragDropTextEdit()
compare_browse = QPushButton("Browse for your Comparison Diagnostic log file")
compare_browse.setAccessibleName("browse")
compare_browse.clicked.connect(lambda: self.browse_file(self.compare_text))
compare_layout.addWidget(compare_label)
compare_layout.addWidget(self.compare_text)
compare_layout.addWidget(compare_browse)
file_layout.addLayout(base_layout)
file_layout.addLayout(compare_layout)
main_layout.addLayout(file_layout)
# Compare button
compare_button = QPushButton("Compare the Base and Comparision log files")
compare_button.clicked.connect(self.compare_packages)
main_layout.addWidget(compare_button)
main_layout.addSpacing(20)
# Results table
self.result_table = QTableWidget()
self.result_table.setColumnCount(3)
self.result_table.setHorizontalHeaderLabels(["Package", "Base Version", "Comparison Version"])
self.result_table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch)
main_layout.addWidget(self.result_table)
main_layout.addSpacing(20)
lower_section_layout = QHBoxLayout()
# Pip commands box
pip_layout = QVBoxLayout()
pip_label = QLabel("Pip commands to align versions:")
pip_layout.addWidget(pip_label)
self.pip_commands = QTextEdit()
self.pip_commands.setReadOnly(True)
pip_layout.addWidget(self.pip_commands)
pip_buttons_layout = QHBoxLayout()
copy_button = QPushButton("Copy Commands")
copy_button.clicked.connect(self.copy_pip_commands)
run_pip_button = QPushButton("Run Pip Commands")
run_pip_button.clicked.connect(self.run_pip_commands)
pip_buttons_layout.addWidget(copy_button)
pip_buttons_layout.addWidget(run_pip_button)
pip_layout.addLayout(pip_buttons_layout)
lower_section_layout.addLayout(pip_layout)
# Add some spacing between the two sections
lower_section_layout.addSpacing(20)
# Organized results box
org_layout = QVBoxLayout()
org_label = QLabel("Organized Results:")
org_layout.addWidget(org_label)
self.org_results = QTextEdit()
self.org_results.setReadOnly(True)
org_layout.addWidget(self.org_results)
copy_org_button = QPushButton("Copy Organized Results")
copy_org_button.clicked.connect(self.copy_organized_results)
org_layout.addWidget(copy_org_button)
lower_section_layout.addLayout(org_layout)
# Add the lower section layout to the main layout
main_layout.addLayout(lower_section_layout)
self.setLayout(main_layout)
self.setGeometry(100, 100, 1200, 800)
self.setWindowTitle('Advanced Package Comparison Tool')
self.show()
def browse_file(self, text_edit):
file_name, _ = QFileDialog.getOpenFileName(self, "Open File", "", "Text Files (*.txt);;Log Files (*.log);;All Files (*)")
if file_name:
try:
# Check if file exists and is not empty
if not os.path.exists(file_name):
raise FileNotFoundError(f"File does not exist: {file_name}")
file_size = os.path.getsize(file_name)
print(f"File size: {file_size} bytes")
if file_size == 0:
print(f"Warning: File is empty: {file_name}")
return
# Try different encoding options
encodings = ['utf-8', 'utf-8-sig', 'ascii', 'latin-1']
content = ""
for encoding in encodings:
try:
with open(file_name, 'r', encoding=encoding) as file:
content = file.read()
print(f"File opened successfully with {encoding} encoding: {file_name}")
break
except UnicodeDecodeError:
print(f"Failed to open with {encoding} encoding, trying next...")
if not content:
raise ValueError("Unable to read file content with any encoding")
text_edit.setText(content)
print(f"Content length: {len(content)} characters")
# Print first few lines for debugging
print("First few lines of content:")
print("\n".join(content.split('\n')[:5]))
except Exception as e:
print(f"Error opening file {file_name}: {str(e)}")
traceback.print_exc()
def extract_packages(self, text):
packages = {}
try:
start_index = text.find("PYTHON PACKAGES:")
if start_index != -1:
package_section = text[start_index:]
lines = package_section.split('\n')[1:] # Skip the "PYTHON PACKAGES:" line
for line in lines:
if line.strip() == "":
break # Stop at the first empty line after packages
parts = line.split('=')
if len(parts) == 2:
package_name = parts[0].strip()
version = parts[1].strip()
packages[package_name.lower()] = version
# print(f"Extracted {len(packages)} packages")
except Exception as e:
print(f"Error extracting packages: {str(e)}")
traceback.print_exc()
return packages
def compare_packages(self):
base_text = self.base_text.toPlainText()
compare_text = self.compare_text.toPlainText()
base_packages = self.extract_packages(base_text)
compare_packages = self.extract_packages(compare_text)
self.result_table.setRowCount(0)
pip_commands = []
missing = []
different = []
matching = []
additional = []
all_packages = sorted(set(base_packages.keys()) | set(compare_packages.keys()), key=str.lower)
for package_lower in all_packages:
row_position = self.result_table.rowCount()
self.result_table.insertRow(row_position)
# Find the original capitalization
package = next((p for p in base_packages.keys() if p.lower() == package_lower),
next((p for p in compare_packages.keys() if p.lower() == package_lower), package_lower))
base_version = base_packages.get(package_lower, "N/A")
compare_version = compare_packages.get(package_lower, "N/A")
self.result_table.setItem(row_position, 0, QTableWidgetItem(package))
self.result_table.setItem(row_position, 1, QTableWidgetItem(base_version))
self.result_table.setItem(row_position, 2, QTableWidgetItem(compare_version))
if base_version == compare_version:
for col in range(3):
self.result_table.item(row_position, col).setBackground(QColor("lightgreen"))
matching.append(f"{package}: {base_version}")
elif base_version == "N/A":
for col in range(3):
self.result_table.item(row_position, col).setBackground(QColor("lightblue"))
additional.append(f"{package}: {compare_version}")
elif compare_version == "N/A":
for col in range(3):
self.result_table.item(row_position, col).setBackground(QColor("pink"))
pip_commands.append(f"pip install {package}=={base_version}")
missing.append(f"{package}: {base_version}")
else:
for col in range(3):
self.result_table.item(row_position, col).setBackground(QColor("yellow"))
pip_commands.append(f"pip install {package}=={base_version}")
different.append(f"{package}: {base_version} (Base) vs {compare_version} (Compare)")
self.pip_commands.setText("\n".join(pip_commands))
organized_results = "Missing in Comparison (present in Base):\n" + "\n".join(sorted(missing)) + "\n\n"
organized_results += "Different Versions:\n" + "\n".join(sorted(different)) + "\n\n"
organized_results += "Matching Versions:\n" + "\n".join(sorted(matching)) + "\n\n"
organized_results += "Additional in Comparison (not in Base):\n" + "\n".join(sorted(additional))
self.org_results.setText(organized_results)
def copy_pip_commands(self):
clipboard = QApplication.clipboard()
clipboard.setText(self.pip_commands.toPlainText())
def copy_organized_results(self):
clipboard = QApplication.clipboard()
clipboard.setText(self.org_results.toPlainText())
def run_pip_commands(self):
commands = self.pip_commands.toPlainText().split('\n')
for command in commands:
try:
subprocess.run(command, shell=True, check=True)
print(f"Successfully executed: {command}")
except subprocess.CalledProcessError as e:
print(f"Error executing {command}: {e}")
def cleanup_logging():
logging.shutdown()
# Global flag to indicate if the application should exit
should_exit = False
def signal_handler(signum, frame):
global should_exit
print("\nCtrl+C pressed. Closing the application.")
should_exit = True
# Set up the signal handler at the global level
signal.signal(signal.SIGINT, signal_handler)
def show_menu():
print("\n\n\n\033[94m AllTalk Diagnostics Tool Menu\033[0m")
print("\n 1. \033[93mGenerate a \033[92mdiagnostics.log\033[93m file\033[0m")
print(" This option creates a new detailed log file of your system's configuration,")
print(" including OS, hardware, Python environment, and installed packages. Use this")
print(" to troubleshoot issues or when seeking support. Old diagnostic.log files will")
print(" be overwritten.")
print("\n 2. \033[93mCompare two \033[92mdiagnostics.log\033[93m logs\033[0m")
print(" This opens a graphical tool to compare two diagnostics.log files. Useful")
print(" for identifying differences between working and non-working configurations")
print(" or for comparing system setups.")
print("\n 9. \033[91mExit\033[0m")
print(" Close the diagnostics tool.")
choice = input("\n Enter your choice (1, 2 or exit with 9): ")
return choice
def main():
print(f"\n\033[94m _ _ _ \033[1;35m_____ _ _ \033[0m _____ _____ ____ ")
print(f"\033[94m / \ | | |\033[1;35m_ _|_ _| | | __ \033[0m |_ _|_ _/ ___| ")
print(f"\033[94m / _ \ | | |\033[1;35m | |/ _` | | |/ / \033[0m | | | | \___ \ ")
print(f"\033[94m / ___ \| | |\033[1;35m | | (_| | | < \033[0m | | | | ___) |")
print(f"\033[94m /_/ \_\_|_|\033[1;35m |_|\__,_|_|_|\_\ \033[0m |_| |_| |____/ ")
app = QApplication(sys.argv)
# Use a timer to check for the exit flag
timer = QTimer()
timer.timeout.connect(lambda: check_exit(app))
timer.start(100) # Check every 100ms
while not should_exit:
choice = show_menu()
if choice == '1':
setup_logging()
log_system_info()
print("\n Diagnostics completed. Check the \033[92mdiagnostics.log\033[0m file for details.\n")
input(" Press Enter to return to the main menu...")
elif choice == '2':
try:
ex = PackageComparisonTool()
ex.show()
app.exec()
except Exception as e:
print(f"Error initializing PackageComparisonTool: {str(e)}")
traceback.print_exc()
elif choice == '9':
cleanup_logging()
print("\n Exiting...")
break
else:
print(" Invalid choice. Please try again.")
cleanup_logging()
app.quit()
def check_exit(app):
if should_exit:
cleanup_logging()
app.quit()
if __name__ == "__main__":
log_system_info()
sys.exit(1)
main()