# dirsync.py - Recursively syncronizes local and remote folders.  Combines functionality
#                    of dirsync and filesync
#
# Input data:  Local and remote backup folders (uses mapped drives)
#
# Output data:  None
#
# System Requirements:  It should run cross-platform.
#
# Revision Log
# Date        Ver    Remarks
#================================================================================================
# 09/27/2007  0.12   Original Coding, derived from filesync, v0.11
# 09/28/2007  0.13   Added some code to assures that both the local and remote store inputs end
#                      in separator characters so that prefix replacement works correctly when
#                      creating the lists of folders to create and delete on the remote store.
# 12/24/2007  0.50   Combined dirsync v0.13 and filesync v0.20, to create a single program 
#                      that doesn't require alteration of batch files when folders are added or
#                      deleted from the local store.  Removed the "pattern matching" from filesync  
#                      code as it was never used.  Also did some serious cleanup of filesync code,
#                      eliminating some archaic methods, mainly with file copying and deletion.
# 12/25/2007  0.51   Added local and remote store folders to the folder pairs data structure for
#                      syncronizing files.  Forgot to sync this level.
# 03/02/2008  0.52   Added a check for updated files for xfer, rather than just whether is exists in
#                      both places.
# 03/20/2008  0.53   After Syncing folder structures, we weren't disciminating against folders when
#                      making local and remote file lists prior to syninc each local and remote
#                      folder.  Folders were slipping into the file lists and when the program tried
#                      delete or copy a folder using a "file" copy method, it crashed.  Don't know
#                      why this wasn't a problem earlier.  Worked on 02/29/2008, was broken on
#                      03/01/2008.  My only guess is that maybe a Microsoft patch was applied to
#                      the system around then that changed the OS behaviour underlying Python and
#                      caused things to break.
# 11/13/2008  0.54   Files that hadn't changed were being transferred every run.  Cause was a 1 second
#                      timestamp difference between systems that made the program think that the
#                      localstore file had changed.  To eliminate this, and deal with any DST time change
#                      wierdness, assumed that ifthe localstore and remotestore timestamps were no more
#                      than 65 minutes apart, then the local store file hasn't changed.  May be overkill,
#                      but few filechanges should be less than an hour apart (at lteast for my purposes).
#
#
####TODO:
##
#Imports
import shutil, sys, string, os, time
#
# Store program name and version for inclusion into the standard banner
program='dirsync, v0.54,'
# Print initial banner
print program, 'Copyright (C) 2008, Tim Sharpe <tim.sharpe@us.army.mil>.'
print 'This program comes with ABSOLUTELY NO WARRANTY and is covered by the'
print 'GNU General Public License.  See the "license.txt" file for details.'
print 'This is free software, and you are welcome to redistribute it under' 
print 'conditions of the GNU General Public License.'
print
#
#
# Define a function to sort lists by string length
def bylength(str1, str2):
    """
    write your own compare function:
    returns value > 0 of word1 longer then word2
    returns value = 0 if the same length
    returns value < 0 of word2 longer than word1
    """
    return len(str1) - len(str2)
#
#
# Main Program
#Check for command-line arguments
if len(sys.argv) <> 3:
    print
    print 'dirsync -- Recursively syncronizes local and remote backup folders, using existing drive letters'
    print
    print 'dirsync  <localpath> <remotepath>'
    print 
    print 'where:    <localpath> = Name of local backup folder.'
    print '         <remotepath> = Name of remote backup folder.'
    print
else:
    #
    # Gather command line arguments
    # Start with the local path
    local_store=sys.argv[1]
    try:
        os.listdir(local_store)
    except:
        print ('ERROR: Invalid local backup folder.  Exiting....')
        sys.exit()
    # Remote path
    remote_store=sys.argv[2]
    try:
        os.listdir(remote_store)
    except:
        print ('ERROR: Invalid remote backup folder.  Exiting....')
        sys.exit()
    #
#
# Clean up local and remote store variables - make sure they end in the separator
if local_store[-1:] != os.sep: local_store+=os.sep
if remote_store[-1:] != os.sep: remote_store+=os.sep
print 'Local file store:', local_store
print 'Remote file store:', remote_store
print
#
folders=[]
local_folders=[]
remote_folders=[]
remote_folders_to_create=[]
remote_folders_to_delete=[]
#
# Obtain the list of folders in the local store
dirwalk=os.walk(local_store)
for root, dirs, names in dirwalk:
    folders.append(root)
for folder in folders:
    if folder!=local_store:
         local_folders.append(folder)
print 'Existing Local Folders: ', local_folders,'\n'
#
# Obtain the list of folders in the remote store
folders=[]
dirwalk=os.walk(remote_store)
for root, dirs, names in dirwalk:
    folders.append(root)
for folder in folders:
    if folder!=remote_store:
         remote_folders.append(folder)
folders=None
print 'Existing Remote Folders: ', remote_folders,'\n'
#
# Determine what new folders need to be created on the remote store
for folder in local_folders:
    if folder.replace(local_store,remote_store) not in remote_folders:
        # Before appending, replace the local_store prefix with the remote_store prefix
        #   We most also remove the separator after the local_store prefix as the remote store already has one
        remote_folders_to_create.append(folder.replace(local_store,remote_store))
print 'Remote Folders To Create: ', remote_folders_to_create,'\n'
#
# Determine what remote folders are no longer valid and need to be deleted
for folder in remote_folders:
    # Before comparing, post fix the seaprator to the local_store prefix
    if folder.replace(remote_store,local_store) not in local_folders:
        remote_folders_to_delete.append(folder)
print 'Remote Folders To Delete: ', remote_folders_to_delete,'\n'
#
# Time to act
# Delete invalid folders from the remote_store
# Sort the delete list by length, then reverse to delete the longest strings first,
#   so that we'll remove the deepest folders first to avoid errors
remote_folders_to_delete.sort(cmp=bylength)
remote_folders_to_delete.reverse()
if len(remote_folders_to_delete)!=0:
    print 'Deleting Invalid Remote Folders:'
    for folder in remote_folders_to_delete:
        print 'Deleting ' + folder + '...'
        try:
            for x in os.listdir(folder):
                # Wrap this in a try structure and ignore errors.  If it fails, nothing is lost.
                try:
                    print '  Deleting ' + os.path.join(folder,x) + '...'
                    os.remove(os.path.join(folder,x))
                except:
                    pass
        except:
            pass
        # Wrap this in a try structure and ignore errors.  If it fails, nothing is lost.
        try:
            os.removedirs(folder)
        except:
            pass
print
#
# Create new folders on the remote store
# Sort the delete list by length, so that lower level folders are created before deeper ones
remote_folders_to_delete.sort(cmp=bylength)
if len(remote_folders_to_create)!=0:
    print 'Create New Remote Folders:'
    for folder in remote_folders_to_create:
        print 'Creating ' + folder + '...'
        # Wrap this in a try structure and ignore errors.  Have to think about what to do if it fails.
        try:
            os.mkdir(folder)
        except:
            pass
#
print '\nSync of folder structure complete, now sync the files...\n'
#
#
# Create a list of source and destination folders pairs for each local_folders entry
folder_pairs=[]
# Add the "root" level of the local and remote data stores.
folder_pairs.append([local_store,remote_store])
# Add the other folders
for x in local_folders:
    folder_pairs.append([x,x.replace(local_store,remote_store)])
print '\nFolder pairs for file synchronization:',folder_pairs
#
# Clean up garbage
del folders
del local_folders
del remote_folders
del remote_folders_to_create
del remote_folders_to_delete
del local_store
del remote_store
#
# Loop through the folder pairs and sync files for each pair
for pair in folder_pairs:
    print '\n\nSyncing local folder:',pair[0]
    # Obtain the list of files in the local store
    local_files=os.listdir(pair[0])
    local_archives=[]
    for x in range(len(local_files)):
        # Store the entry for action ONLY is it's a file, not a directory.
        if os.path.isfile(os.path.join(pair[0],local_files[x])):
            local_archives.append(local_files[x])
    local_archives.sort()
    local_archives.reverse()
    #
    # Obtain the list of files in the remote store
    remote_files=os.listdir(pair[1])
    remote_archives=[]
    for x in range(len(remote_files)):
        # Store the entry for action ONLY is it's a file, not a directory.
        if os.path.isfile(os.path.join(pair[1],remote_files[x])):
            remote_archives.append(remote_files[x])
    remote_archives.sort()
    remote_archives.reverse()
    #
    print 'Local Archival Backups:',local_archives
    print '\nRemote Archival Backups:',remote_archives
    #
    # Now that we have local archival backups in "local_archives" and remote archival backups
    #   in "remote_archives", we can synchronize the two
    #
    # Delete any archival backups in the remote store that aren't in the local store
    # Create an empty list to hold files to be deleted and fill it with deal soldiers
    file_list=[]
    for x in remote_archives:
        if x not in local_archives:
            file_list.append(x)
    print '\nFiles To Delete From Remote Archive:', file_list
    # If there are any files to delete, delete them now.
    if len(file_list)>0:
        for x in file_list:
            print 'Deleting ' + x + '...'
            os.remove(os.path.join(pair[1],x))
            #
    #
    # Transfer any local archives not aleady in the remote store.  This sends up the new ones.
    #   We also need to verify that we're only sending files and not directories.
    file_list=[]
    newfile_list=[]
    update_list=[]
    for x in local_archives:
        # Check for files that don't exist in the remote store
        if x not in remote_archives and os.path.isfile(os.path.join(pair[0],x))==1:
            file_list.append(x)
            newfile_list.append(x)
        # Check for files that exist on the remote store but have been modified
        #print os.path.join(pair[0],x),os.stat(os.path.join(pair[0],x))[8],os.stat(os.path.join(pair[1],x))[8],abs(os.stat(os.path.join(pair[0],x))[8]-os.stat(os.path.join(pair[1],x))[8])
        #   We've has files being transferred because the time on one system was a second or two off the
        #      other.  We'll assume that is there's less than 65 minutes (3900 seconds) difference in the
        #      timestamps, that the file hasn't been modified (this should also take care of any DST  time
        #      change differences).
        if x in remote_archives and abs(os.stat(os.path.join(pair[0],x))[8]-os.stat(os.path.join(pair[1],x))[8])>3900:
            file_list.append(x)
            update_list.append(x)
    if len(newfile_list)!=0:
        print '\nFiles that exist in the local store but not in the remote store:', newfile_list
    if len(update_list)!=0:
        print '\nFiles that exist in both stores but are newer on the local store:', update_list
    print '\nFiles To Transfer To Remote Archive:', file_list
    # Tranfer the files
    for x in file_list:
        print 'Sending ' + x + '...'
        shutil.copy2(os.path.join(pair[0],x),os.path.join(pair[1],x))
    #
    # Print an updated list of the remote archive 
    remote_files=os.listdir(pair[1])
    remote_archives=[]
    for x in range(len(remote_files)):
        remote_archives.append(remote_files[x])
    remote_archives.sort()
    remote_archives.reverse()
    print '\nRemote Archives After Synchronization:',remote_archives
    #
    #
print '\nAll done!'