Find and Delete Duplicate Files Using Hashing in Python

import hashlib
import os
from pathlib import Path

def file_hash(path, block_size=65536):
    """Return SHA256 hash of file content."""
    hasher = hashlib.sha256()
    with open(path, 'rb') as f:
        while chunk := f.read(block_size):
            hasher.update(chunk)
    return hasher.hexdigest()

def find_and_delete_duplicates(directory):
    """Find and delete duplicate files in directory (recursively)."""
    hashes = {}
    duplicates = []
    for filepath in Path(directory).rglob('*'):
        if filepath.is_file():
            h = file_hash(filepath)
            if h in hashes:
                duplicates.append(filepath)
                print(f"Deleting duplicate: {filepath}")
                filepath.unlink()
            else:
                hashes[h] = filepath
    return duplicates

if __name__ == "__main__":
    # Example: create test files in a temporary directory
    import tempfile
    import time
    with tempfile.TemporaryDirectory() as tmpdir:
        # Create two identical files
        f1 = Path(tmpdir) / "file1.txt"
        f2 = Path(tmpdir) / "file2.txt"
        f1.write_text("Hello World")
        time.sleep(0.1)  # Ensure different timestamps
        f2.write_text("Hello World")
        # Create a unique file
        f3 = Path(tmpdir) / "unique.txt"
        f3.write_text("Unique content")
        dups = find_and_delete_duplicates(tmpdir)
        print(f"Found and deleted {len(dups)} duplicate(s)")
        remaining = list(Path(tmpdir).rglob('*'))
        print(f"Remaining files: {[p.name for p in remaining]}")