SFTP Automation
You can use this python script to mirror all files you have available within IDC SFTP to a location on your machine.
How to use the script:
- Fill-in required information – username, the path to the private key and the password for the private key
- install the latest version of the paramiko package
pip.exe install --upgrade paramiko
- OPTIONAL – Adjust the Configuration, such as download path
- Execute the script
import os
import stat
import paramiko # Third-party SFTP library
# Configuration
SFTP_HOST = 'sftp.idc.com'
SFTP_PORT = 22
REMOTE_ROOT = '/' # Root location within the SFTP server, keep as '/' to download all files
LOCAL_ROOT = 'sftp-download' # Root location within local system where files will be downloaded
SKIP_EXISTING_FILES = True # Do not download files that are already present locally
# Credentials - use a secret store or an environment variable to store the credentials
# Storing those in a script is dangerous and not recmmended
USERNAME = '<username>' # IDC provided username
PRIVATE_KEY_PATH = '/' # Filesystem path to IDC provided SSH key
PRIVATE_KEY_PASSWORD = '<ssh key password>' # IDC provided ssh key password
def connect_sftp():
key = paramiko.RSAKey.from_private_key_file(PRIVATE_KEY_PATH, password=PRIVATE_KEY_PASSWORD)
transport = paramiko.Transport((SFTP_HOST, SFTP_PORT))
transport.connect(username=USERNAME, pkey=key)
return paramiko.SFTPClient.from_transport(transport)
def mirror_dir(sftp, remote_path, local_path):
os.makedirs(local_path, exist_ok=True)
for item in sftp.listdir_attr(remote_path):
remote_item_path = os.path.join(remote_path, item.filename).replace('\\', '/')
local_item_path = os.path.join(local_path, item.filename)
if stat.S_ISDIR(item.st_mode):
mirror_dir(sftp, remote_item_path, local_item_path)
else:
if SKIP_EXISTING_FILES and os.path.exists(local_item_path):
print(f"Skipping existing: {local_item_path}")
continue
print(f"Downloading: {remote_item_path} -> {local_item_path}")
sftp.get(remote_item_path, local_item_path)
def main():
sftp = connect_sftp()
try:
mirror_dir(sftp, REMOTE_ROOT, LOCAL_ROOT)
finally:
sftp.close()
if __name__ == '__main__':
main()
Tip: You can schedule the script to run e.g., every midnight, so that you always have your local folder synchronized with IDC SFTP server.