#!/usr/bin/env python # -*- coding: iso8859-1 -*- # # filechecker.py v0.2 - Erik Waling # (my first python program so don't expect nice code :)) # # Scans a given directory and it's sub-directorys for .sfv-files. # When found the program will check the CRC32 of all files named # in the sfv and check if they match the ones given in the file. # Once a check is done a "tag" is created in the directory. If # all files are ok the program won't touch that directory in the # next scan. When files are missing or bad (according to the CRC32) # the program will write some info in the tag and create 0-byte # files with the same name as the file missing with a added suffix # in the end. Incomplete directorys are continued to be checked # until they are complete. The 0-byte files are removed as soon # as a complete file is present in the directory. # # DISCLAIMER: USE THIS PROGRAM AT YOUR OWN RISK! import sys import os import os.path import fchksum import string import re import time from stat import * filescanner_version = '0.2' # config stuff # tokens that will be replaced in info_tag # # %received% the number of files received # %expected% the number of files expected (according to .sfv) # %size% total size of received files in megabyes # %status% replaced with status_complete or status_incomplete # token that will be replaced in status_incomplete # # %percent_done% the percentage done ( (received/expected)*100 ) tag_prefix = '-[WHATEVER] -' info_tag = '( %received%F %size%M - %status% )' tag_suffix = '- [WHATEVER]-' delete_broken = False # deletes broken files if True crc_fail_prefix = 'CRCFAILED-' # used when delete_broken is False file_missing_suffix = '-missing' status_complete = 'COMPLETE' status_incomplete = 'INCOMPLETE - %percent_done%% DONE' # warning: this string should _not_ be a substring of status_complete! def filescan(directory): for name in os.listdir(directory): path = os.path.join(directory, name) if (os.path.isfile(path) and name.lower().endswith('.sfv') and not directory_done(os.path.dirname(path))): process_sfv(path) elif (os.path.isdir(path)): filescan(path) def get_tags_filename(directory): for name in os.listdir(directory): if (os.stat(os.path.join(directory,name))[ST_SIZE] == 0): if (name.startswith(tag_prefix) and name.endswith(tag_suffix)): match_against = string.split(status_incomplete, "%percent_done%") regex = '.*' for a in match_against: regex = regex + a + '.*' regex_pattern = re.compile(regex) if (re.match(regex_pattern, name)): return os.path.join(directory, name) regex_pattern = re.compile(status_complete) if (re.match(regex_pattern, name)): return os.path.join(directory, name) return '' def directory_done(directory): for name in os.listdir(directory): if (os.stat(os.path.join(directory,name))[ST_SIZE] == 0): if (name.startswith(tag_prefix) and name.endswith(tag_suffix)): match_against = string.split(status_incomplete, "%percent_done%") regex = '.*' for a in match_against: regex = regex + a + '.*' regex_pattern = re.compile(regex) if (re.match(regex_pattern, name)): return False else: return True return False def process_sfv(filename): f = open(filename, 'r') lines = f.readlines() f.close() files_received = 0 file_size = 0 files_expected = 0 make_tag = False directory = os.path.dirname(filename) previous_tag = get_tags_filename(directory) if (len(previous_tag) == 0): make_tag = True print 'New SFV file found: ', filename for line in lines: if not line.startswith(';'): # comments starts with ';' files_expected += 1 # each line that is not a comment is expected to be a file line = line.strip('\n\r') crc = line[len(line)-8:] # grab the CRC32, the last 8 characters fname = line[:len(line)-9] # ... and the filename if (len(previous_tag) == 0): # created files with file_missing_suffix first time directory is scanned create_file(directory, fname + file_missing_suffix) check_file = os.path.join(directory, fname) if (os.path.exists(check_file + file_missing_suffix)): ret = crc32_check(check_file, crc) if (ret == 1): # file ok make_tag = True files_received += 1 file_size += os.path.getsize(check_file)/1024/1024 # filesize in megabyte try: os.remove(check_file + file_missing_suffix) # remove the file indicating that this file is missing except OSError: pass elif (ret == 0): # wrong crc print 'Bad CRC: ', check_file if (delete_broken): print 'Deleting file' os.remove(check_file) else: print 'Moving file to: ' , os.path.join(os.path.dirname(filename), crc_fail_prefix + fname) os.rename(check_file, os.path.join(directory, crc_fail_prefix + fname)) else: # we still need to count the files and their size for the new tag files_received += 1 file_size += os.path.getsize(check_file)/1024/1024 # filesize in megabyte if (make_tag): try: os.remove(previous_tag) #print 'DETETED OLD TAG: ' , previous_tag except OSError: pass tag = create_tag(os.path.dirname(filename), files_expected, files_received, file_size) #print 'New tag created: ', tag create_file(os.path.dirname(filename), tag) if (files_received == files_expected): print 'Directory complete: ', directory def crc32_check(filename, crc): if not (os.path.exists(filename)): return -1 elif (fchksum.fcrc32t(filename)[0].lower() == crc.lower()): return 1 return 0 def create_tag(directory, expected, received, size): tag = string.replace(info_tag, '%received%', str(received)) tag = string.replace(tag, '%expected%', str(expected)) tag = string.replace(tag, '%size%', str(size)) if (expected == received): status = status_complete else: status = string.replace(status_incomplete, '%percent_done%', str(int((float(received)/expected)*100))) tag = string.replace(tag, '%status%', status) tag = tag_prefix + tag + tag_suffix return tag def create_file(directory, filename): f = open(os.path.join(directory, filename), 'w') f.close() if __name__ == '__main__': if (len(sys.argv) < 3): print 'Usage: filechecker.py ' sys.exit(0) print 'filechecker.py v%s\nScanning directorys under: %s\nDelay between scans: %s seconds\n\n' %(filescanner_version, sys.argv[2], sys.argv[1]) while (True): filescan(sys.argv[2]) time.sleep(int(sys.argv[1]))