dupecheck.py

Thu, 04 Oct 2018 00:42:48 +0200

author
mdd
date
Thu, 04 Oct 2018 00:42:48 +0200
changeset 33
83bcb5931ee3
parent 32
df89a8fba2a2
child 35
14c966c10648
permissions
-rwxr-xr-x

function to fix weird filenames

3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
1 #!/usr/bin/env python
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
2 # -*- coding: utf-8 -*-
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
3 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
4 Toolkit / executable to scan for duplicate filenames in movie database
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
5
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
6 2017 by mdd
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
7 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
8
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
9 #pylint: disable=line-too-long
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
10 #pylint: disable=invalid-name
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
11
4
a7e9e7974c22 prepare for speedup
mdd
parents: 3
diff changeset
12 from __future__ import print_function
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
13 import os, sys
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
14
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
15 def similarity(a, b):
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
16 if DIFFLIB:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
17 return difflib.SequenceMatcher(a=a, b=b).ratio()
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
18 else:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
19 return Levenshtein.ratio(a, b)
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
20
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
21 suffixes = ['b', 'K', 'M', 'G', 'T', 'P']
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
22 def humansize(nbytes):
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
23 i = 0
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
24 while nbytes >= 1024 and i < len(suffixes)-1:
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
25 nbytes /= 1024.
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
26 i += 1
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
27 f = ('%.2f' % nbytes).rstrip('0').rstrip('.')
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
28 return '%s %s' % (f, suffixes[i])
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
29
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
30 def replace_all(text, dic):
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
31 for i, j in dic.iteritems():
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
32 text = text.replace(i, j)
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
33 return text
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
34
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
35 class dupechecker(object):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
36 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
37 Simple class to scan multiple directories recursive,
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
38 build a list of movie filenames.
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
39 analyze the list for duplicates and dump them
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
40 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
41 def __init__(self):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
42 self.basedir = ""
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
43 self.filelist = []
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
44 self.duplicates = {}
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
45 self.ratio = 0.85
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
46
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
47
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
48 def reset(self):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
49 self.filelist = []
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
50 self.duplicates = {}
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
51
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
52 def scandir(self, basedir, extra=[]):
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
53 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
54 Scan a base directory for movie files and add them to
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
55 the list for analyze
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
56 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
57 self.basedir = basedir
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
58 print("Scanning directory: %s" % basedir)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
59 for root, subdirs, files in os.walk(basedir):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
60 for filename in files:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
61 ext = os.path.splitext(filename)[1].lower()
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
62 if ext == ".ts":
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
63 #file_path = os.path.join(root, filename)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
64 title = filename.split(" - ")
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
65 if len(title) == 1:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
66 title = title[0]
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
67 else:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
68 title = " - ".join(title[2:])
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
69 title = title[:-3].lower()
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
70 self.filelist.append([title, filename, root, ext])
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
71 elif ext in ['.mkv', '.avi', '.mpg', '.mpeg', '.mp4']:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
72 title = filename[:-4].lower()
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
73 self.filelist.append([title, filename, root, ext])
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
74 elif ext in extra:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
75 title = filename[:-4].lower()
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
76 self.filelist.append([title, filename, root, ext])
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
77
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
78 def fixnames(self):
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
79 """
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
80 Search for defect filenames and remove illegal characters
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
81 """
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
82 import re
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
83 for item in self.filelist:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
84 if not item[3] in ['.mkv', '.txt']:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
85 continue
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
86 # any non-alphanumeric characters in filename?
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
87 cleanfn = replace_all(item[1], {
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
88 #'ä':'ae', 'Ä':'Ae',
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
89 #'ö':'oe', 'Ö':'Oe',
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
90 #'ü':'ue', 'Ü':'Ue',
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
91 'ß':'ss',
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
92 })
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
93 cleanfn = re.sub(r'[^A-Za-z0-9\.\_\-\(\)\&öäüÖÄÜ\' ]', '-', cleanfn)
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
94 if item[1] == cleanfn:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
95 continue
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
96 print (item[1])
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
97 os.rename(
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
98 os.path.join(item[2], item[1]),
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
99 os.path.join(item[2], cleanfn)
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
100 )
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
101
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
102 def statistics(self):
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
103 """
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
104 Summarize disk usage and print stats about found filetypes
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
105 """
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
106 stats = {}
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
107 for item in self.filelist:
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
108 if not item[3] in stats:
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
109 stats[item[3]] = [0, 0.0]
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
110 stats[item[3]][0] += 1
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
111 stats[item[3]][1] += os.stat(
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
112 os.path.join(
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
113 item[2], item[1])).st_size
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
114 print ("%5s %6s %10s" % (
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
115 "File:",
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
116 "Count:",
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
117 "Size:"))
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
118 sum_count = 0
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
119 sum_size = 0.0
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
120 for ext in stats.keys():
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
121 sum_count += stats[ext][0]
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
122 sum_size += stats[ext][1]
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
123 print ("%5s %6i %10s" % (
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
124 ext, stats[ext][0],
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
125 humansize(stats[ext][1])))
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
126 print ("%5s %6i %10s" % (
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
127 "TOTAL", sum_count,
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
128 humansize(sum_size)))
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
129
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
130
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
131 def analyze(self):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
132 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
133 Analyze the scanlist for duplicates
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
134 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
135 print("%i files to analyze, running duplicate testing loop..." % (
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
136 len(self.filelist)))
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
137
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
138 listlen = len(self.filelist)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
139 for idx in range(listlen):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
140 if not self.filelist[idx]:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
141 continue
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
142 print("\r%d %s\033[K" % (
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
143 idx, self.filelist[idx][0]), end='')
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
144 sys.stdout.flush()
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
145 for idx2 in range(idx + 1, listlen):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
146 if self.filelist[idx2]:
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
147 if similarity(self.filelist[idx][0], self.filelist[idx2][0]) > self.ratio:
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
148 #print "possible duplicate %d %s" % (idx2, item2[0])
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
149 key = os.path.join(self.filelist[idx][2], self.filelist[idx][1])
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
150 if not key in self.duplicates:
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
151 self.duplicates[key] = []
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
152 self.duplicates[key].append(
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
153 os.path.join(
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
154 self.filelist[idx2][2],
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
155 self.filelist[idx2][1]
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
156 ))
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
157 # unset the found duplicate, so that this will not be scanned again
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
158 self.filelist[idx2] = None
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
159 print("\n\n")
3
569fa9a431b9 added filename duplicate checker
mdd
parents:
diff changeset
160
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
161 def output(self):
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
162 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
163 Dump found duplicates to console
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
164 """
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
165 idx = 1
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
166 for base in self.duplicates:
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
167 print("Duplicate file set #%i" % idx)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
168 print(base)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
169 for dup in self.duplicates[base]:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
170 print(dup)
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
171 print()
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
172 idx += 1
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
173
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
174
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
175 if __name__ == "__main__":
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
176 # parse command line options
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
177 import argparse
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
178
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
179 parser = argparse.ArgumentParser(\
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
180 description='Movie database filename duplicate checker')
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
181 parser.add_argument('--ratio', type=float, default=0.85, \
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
182 help='filename duplicate threshold 0.1 < ratio 1.0 (default 0.85)')
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
183 parser.add_argument('--difflib', action='store_true', default=False, \
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
184 help='force the use of difflib instead Levenshtein')
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
185 parser.add_argument('--stats', action='store_true', default=False, \
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
186 help='generate stats summary instead of check for duplicates')
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
187 parser.add_argument('--fixnames', action='store_true', default=False, \
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
188 help='scan for mkv and txt, fix broken filenames for windows')
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
189 parser.add_argument('basedir', metavar='basedir', nargs='+', \
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
190 help='one or more base directories')
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
191
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
192 args = parser.parse_args()
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
193 dupe = dupechecker()
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
194 dupe.ratio = args.ratio
22
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
195 if args.difflib:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
196 DIFFLIB = True
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
197 import difflib
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
198 else:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
199 try:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
200 import Levenshtein
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
201 DIFFLIB = False
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
202 except ImportError:
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
203 import difflib
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
204 DIFFLIB = True
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
205 print("Consider 'pip install python-Levenshtein' for faster analyze")
c18abd9198c0 implemented Levenshtein algorithm for incredible speedup
mdd
parents: 21
diff changeset
206
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
207
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
208 if args.fixnames:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
209 for srcstr in args.basedir:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
210 dupe.scandir(srcstr, ['.txt'])
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
211 if len(dupe.filelist) > 0:
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
212 print ("Checking %i file names..." % len(dupe.filelist))
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
213 dupe.fixnames()
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
214 dupe.filelist = []
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
215 sys.exit(0)
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
216
21
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
217 for srcstr in args.basedir:
1c0beeca2f9c cleanup dupechecker
mdd
parents: 15
diff changeset
218 dupe.scandir(srcstr)
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
219
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
220 if args.stats or args.fixnames:
32
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
221 dupe.statistics()
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
222 else:
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
223 dupe.analyze()
df89a8fba2a2 added stats calc to dupechecker
mdd
parents: 22
diff changeset
224 dupe.output()
33
83bcb5931ee3 function to fix weird filenames
mdd
parents: 32
diff changeset
225

mercurial