251 lines
10 KiB
Python
251 lines
10 KiB
Python
"""
|
||
cksgen – Generate and compare MD5 checksum lists with Python
|
||
|
||
Copyright (c) 2023 Helmut Kaczmarek <code@helmutkaczmarek.de>
|
||
|
||
This program is free software: you can redistribute it and/or modify
|
||
it under the terms of the GNU General Public License as published by
|
||
the Free Software Foundation, either version 3 of the License, or
|
||
(at your option) any later version.
|
||
|
||
This program is distributed in the hope that it will be useful,
|
||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
GNU General Public License for more details.
|
||
|
||
You should have received a copy of the GNU General Public License
|
||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
||
General Usage:
|
||
cksgen [-h] [-c CONFIG] [-e]
|
||
|
||
If you haven't compiled the cksgen.py file into an executable file (e.g.
|
||
cksgen.exe on a Windows system; you can do so with "pyinstaller
|
||
--onefile cksgen.py")) and haven't registered it in the system
|
||
environment variables, then:
|
||
|
||
python /path/to/cksgen.py [-h] [-c CONFIG] [-e]
|
||
|
||
Help:
|
||
cksgen -h
|
||
|
||
See example configuration file:
|
||
cksgen -e
|
||
|
||
Example usage with a configuration file "photos.conf":
|
||
cksgen -c photos
|
||
"""
|
||
|
||
import os
|
||
import hashlib
|
||
import glob
|
||
import datetime
|
||
import configparser
|
||
import argparse
|
||
|
||
# Example configuration that users can see with "-e" option
|
||
EXAMPLE_CONFIG = r"""
|
||
# Configuration file for cksgen.py
|
||
#
|
||
# Save the configuration files in the same
|
||
# folder where the program is located.
|
||
[USER_SETTINGS]
|
||
# The configuration's name.
|
||
# A subfolder with this name will be created.
|
||
conf_name = Example
|
||
|
||
# File types that cksgen will look for.
|
||
# Multiple file types are separated by commas without spaces.
|
||
allowed_extensions = jpg,jpeg,dng,cr2
|
||
|
||
# The actual directory where cksgen looks for files.
|
||
data_directory = C:\Users\Username\Path\To\Example
|
||
|
||
# The directory where lists of MD5 checksums are placed.
|
||
lists_directory = conf_name\Lists
|
||
|
||
# The directory in which the log files are stored.
|
||
logs_directory = conf_name\Logs
|
||
|
||
# Specifies how many checksum files and log files cksgen should keep.
|
||
files_to_keep = 10
|
||
"""
|
||
|
||
# Parse command line arguments
|
||
parser = argparse.ArgumentParser(description="Generate MD5 checksums for files in a directory.")
|
||
parser.add_argument("-c", "-conf", "--config", help="Specify the configuration file to use.")
|
||
parser.add_argument("-e", "-ex", "--exampleconf", action="store_true", help="Display example configuration.")
|
||
args = parser.parse_args()
|
||
|
||
# Display example configuration and exit if "-e" option is used
|
||
if args.exampleconf:
|
||
print(EXAMPLE_CONFIG)
|
||
exit(0)
|
||
|
||
# Process configuration if "-c" option is used
|
||
if args.config:
|
||
if args.config == 'example':
|
||
config_content = EXAMPLE_CONFIG
|
||
else:
|
||
print("Starting cksgen...")
|
||
config_filename = args.config + ".conf"
|
||
with open(config_filename, 'r') as config_file:
|
||
config_content = config_file.read()
|
||
|
||
# Parse the configuration content
|
||
config = configparser.ConfigParser()
|
||
config.read_string(config_content)
|
||
|
||
# Extract user settings from the configuration
|
||
allowed_extensions = [ext.strip() for ext in config.get('USER_SETTINGS', 'allowed_extensions').split(';')]
|
||
data_directory = config.get('USER_SETTINGS', 'data_directory')
|
||
lists_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'lists_directory'))
|
||
logs_directory = os.path.join(os.path.dirname(__file__), config.get('USER_SETTINGS', 'logs_directory'))
|
||
files_to_keep = int(config.get('USER_SETTINGS', 'files_to_keep'))
|
||
|
||
# Function to load configuration from a file
|
||
def load_config(config_path):
|
||
config = configparser.ConfigParser()
|
||
config.read(config_path)
|
||
return config['USER_SETTINGS']
|
||
|
||
# Function to delete old files based on number to keep
|
||
def delete_old_files(directory, files_to_keep):
|
||
all_files = glob.glob(os.path.join(directory, '*.*'))
|
||
all_files.sort(key=os.path.getmtime, reverse=True)
|
||
|
||
files_to_delete = all_files[files_to_keep:]
|
||
for file in files_to_delete:
|
||
os.remove(file)
|
||
|
||
# Function to compare checksum files for differences
|
||
def compare_checksum_files(file1, file2):
|
||
with open(file1, 'r') as f1, open(file2, 'r') as f2:
|
||
checksums1 = f1.read().splitlines()[1:] # Skip the first line
|
||
checksums2 = f2.read().splitlines()[1:] # Skip the second line
|
||
|
||
different_files = []
|
||
|
||
if not checksums2:
|
||
return ["<single_file>"]
|
||
|
||
for line1 in checksums1:
|
||
checksum1, file1 = line1.split('\t')
|
||
for line2 in checksums2:
|
||
checksum2, file2 = line2.split('\t')
|
||
if file1 == file2 and checksum1 != checksum2:
|
||
different_files.append(file1)
|
||
break
|
||
|
||
return different_files
|
||
|
||
# Main execution block
|
||
if __name__ == "__main__":
|
||
# Parse command line arguments
|
||
parser = argparse.ArgumentParser(description="Generate checksums for files.")
|
||
parser.add_argument("-conf", "--config", type=str, help="Configuration file name without extension")
|
||
|
||
args = parser.parse_args()
|
||
|
||
# If a configuration file is specified
|
||
if args.config:
|
||
config_filename = args.config + ".conf"
|
||
config = load_config(config_filename)
|
||
|
||
# Extract settings from the configuration
|
||
conf_name = config.get('conf_name')
|
||
allowed_extensions = config.get('allowed_extensions').split(',')
|
||
data_directory = config.get('data_directory')
|
||
files_to_keep = int(config.get('files_to_keep'))
|
||
|
||
# Get the script's directory
|
||
script_directory = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
# Create paths for lists and logs directories
|
||
lists_directory = os.path.join(script_directory, conf_name, 'Lists')
|
||
logs_directory = os.path.join(script_directory, conf_name, 'Logs')
|
||
|
||
# Create the directories if they don't exist
|
||
os.makedirs(lists_directory, exist_ok=True)
|
||
os.makedirs(logs_directory, exist_ok=True)
|
||
|
||
# Generate timestamp and datetime strings
|
||
current_timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
current_datetime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
# Generate filenames for checksum and log files
|
||
checksum_filename = os.path.join(lists_directory, f'{current_timestamp}_checksum.txt')
|
||
log_filename = os.path.join(logs_directory, f'{current_timestamp}_log.txt')
|
||
|
||
# Function to scan a directory and generate checksums
|
||
def scan_directory(directory):
|
||
with open(checksum_filename, 'a') as f:
|
||
f.write(f"MD5 checksums on: {current_datetime}\n")
|
||
|
||
# Walk through the directory tree
|
||
for root, _, files in os.walk(directory):
|
||
for file in files:
|
||
file_path = os.path.join(root, file)
|
||
extension = file_path.split('.')[-1].lower()
|
||
|
||
# Calculate checksum for allowed file types
|
||
if extension in allowed_extensions:
|
||
md5_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest()
|
||
|
||
# Write checksum and file path to the checksum file
|
||
with open(checksum_filename, 'a') as f:
|
||
f.write(f'{md5_checksum}\t{file_path}\n')
|
||
|
||
# Display progress
|
||
print(f'Processing {file_path}')
|
||
|
||
# Scan the specified data directory
|
||
scan_directory(data_directory)
|
||
|
||
# Create a log entry for successful checksum generation
|
||
log_entry = f'MD5 checksums have been created and stored in {checksum_filename}.\n'
|
||
with open(log_filename, 'a') as f:
|
||
f.write(log_entry)
|
||
|
||
# Delete old files based on the specified number to keep
|
||
delete_old_files(lists_directory, files_to_keep)
|
||
delete_old_files(logs_directory, files_to_keep)
|
||
|
||
# Compare the last two checksum files for differences
|
||
last_checksum_files = glob.glob(os.path.join(lists_directory, '*_checksum.txt'))
|
||
if len(last_checksum_files) >= 2:
|
||
# Sort the list of files in reverse order (newest first)
|
||
last_checksum_files.sort(reverse=True)
|
||
|
||
# Get the paths of the two most recent checksum files
|
||
last_checksum_file1 = last_checksum_files[0]
|
||
last_checksum_file2 = last_checksum_files[1]
|
||
|
||
# Compare the checksums in the two files
|
||
different_files = compare_checksum_files(last_checksum_file1, last_checksum_file2)
|
||
|
||
log_message = ''
|
||
|
||
# If there are different checksums, log and print the differences
|
||
if different_files:
|
||
print('ATTENTION: Different MD5 checksums found! See log file in', log_filename)
|
||
log_message += 'ATTENTION: The following files have different checksums:\n'
|
||
for file in different_files:
|
||
log_message += file + '\n'
|
||
else:
|
||
# If no differences, log and print a message
|
||
log_message += 'INFO: No different MD5 checksums found.\n'
|
||
print('INFO: No different MD5 checksums found.')
|
||
|
||
# Write the log message to the log file
|
||
with open(log_filename, 'a') as f:
|
||
f.write(log_message)
|
||
|
||
# If there's only one checksum file, log and print a message
|
||
elif len(last_checksum_files) == 1:
|
||
with open(log_filename, 'a') as f:
|
||
f.write("INFO: Checksums could not be compared because there is currently only one checksum file.\n")
|
||
|
||
print("INFO: Checksums could not be compared because there is currently only one checksum file.")
|
||
|
||
print(log_entry) |