취미로 코딩하기

// -----------------------------------------------------------------------------
// 100MB 이상의 파일중에서 중복된 파일 찾기
// -----------------------------------------------------------------------------
 
import java.security.MessageDigest
 
final TARGET_DIR = "C:\\"
final LIMIT_SIZE = 100000000
 
def md5sum(final file) {
    MessageDigest digest = MessageDigest.getInstance("MD5")
    file.withInputStream() { is ->          
        byte[] buffer = new byte[8192]
        int read = 0
        while( (read = is.read(buffer)) > 0) {
            digest.update(buffer, 0, read);
        }
    }                                                        
    byte[] md5 = digest.digest()
    BigInteger bigInt = new BigInteger(1, md5)
    return bigInt.toString(16).padLeft(32, '0')
}
 
def file_list     = []
def distinct_list = [:]
def startDate = new Date().format('yyyy/MM/dd HH:mm:ss')
 
new File(TARGET_DIR).eachFileRecurse { file -> 
    if( file.size() > LIMIT_SIZE ) {
        def md5 = null
        try {
            md5 = md5sum(file)
        } catch(ex) {
            md5 = null
        }
        if (md5 != null) { 
            if(distinct_list[md5] == null)
                distinct_list[md5] = 1
            else
                distinct_list[md5] += 1
            file_list << (md5+"|"+file)
        }
    }
}
 
distinct_list = distinct_list.sort { a, b -> b.value <=> a.value }
 
distinct_list.each { md5, cnt ->
    if( cnt > 1) {
       println "\n[ $md5 ]"
       file_list.each { file -> 
           def (md5_2, filename) = file.split(/\|/)
           if(md5 == md5_2) { println filename }
       }
    }
}
 
def endDate = new Date().format('yyyy/MM/dd HH:mm:ss')
println "\n\n\n"
println "================================================================================"
println "Start Time : $startDate"
println "End   Time : $endDate"
println "================================================================================"

import java.security.MessageDigest
 
def md5sum(final file) {
    MessageDigest digest = MessageDigest.getInstance("MD5")
    file.withInputStream() { is ->          
        byte[] buffer = new byte[8192]
        int read = 0
        while( (read = is.read(buffer)) > 0) {
            digest.update(buffer, 0, read);
        }
    }                                                        
    byte[] md5 = digest.digest()
    BigInteger bigInt = new BigInteger(1, md5)
    return bigInt.toString(16).padLeft(32, '0')
}
 
println md5sum(new File("md5sum.groovy"))