PHP, Ruby 에 이어서, Python 에서도 중복파일 찾기를 만들어보았다.

# -*- coding: cp949 -*-
# 버전 3.2.2 이상
 
from operator import itemgetter
from hashlib import md5
import os
 
TARGET_DIR   = "M:\\PATH\\TO\\특정디렉토리"
LIMITED_SIZE = 10*(1024*1024) # 10MB
 
 
def md5sum(filename, buf_size=4068):
    m = md5()
    with open(filename, 'rb') as f:
        data = f.read(buf_size)
        while data:
            m.update(data)
            data = f.read(buf_size)
    return m.hexdigest()
 
 
def main():
    hash_cnt  = {}
    file_list = []
    for p, ds, fs in os.walk(TARGET_DIR):
        for f in fs:
            filename = os.path.join(p, f)
            if not os.path.isfile(filename) : continue
            if os.path.islink(filename) : continue
            if os.path.getsize(filename) < LIMITED_SIZE: continue
            crc = md5sum(filename)
            if crc in hash_cnt:
                hash_cnt[crc] = hash_cnt[crc] + 1
            else:
                hash_cnt[crc] = 1
            file_list.append(crc+"|"+filename)
 
    for hash, cnt in sorted(hash_cnt.items(), key=itemgetter(1), reverse=True):
        if cnt < 2: continue
        print("\n["+hash+"]")
        for item in file_list:
            (hash2, filename) = item.split("|")
            if hash == hash2 : print(filename)
 
 
if __name__ == '__main__':
    main()

+ Recent posts