summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFilipp Lepalaan <filipp@mekanisti.fi>2009-10-20 06:13:04 +0300
committerFilipp Lepalaan <filipp@mekanisti.fi>2009-10-20 06:13:04 +0300
commitfa6720574520719540bdfc0044ddbb58dece147c (patch)
tree138894455f6e54f7eceda881e7d219a7cc8bc5b3
downloadMediaRelay-master.tar.gz
MediaRelay-master.tar.bz2
MediaRelay-master.zip
Initial commitHEADmaster
-rw-r--r--config.yaml3
-rw-r--r--relay.py117
2 files changed, 120 insertions, 0 deletions
diff --git a/config.yaml b/config.yaml
new file mode 100644
index 0000000..bd2ccdb
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1,3 @@
+- default:
+ source: user:pass@server/folder
+ destination: user:pass@otherserver/folder
diff --git a/relay.py b/relay.py
new file mode 100644
index 0000000..495e6e9
--- /dev/null
+++ b/relay.py
@@ -0,0 +1,117 @@
+"""
+ MediaRelay/relay.py
+ Uploads new FTP files to other server.
+ This doesn't synchronize stuff, just copies new things from A to B
+ @author Filipp Lepalaan <filipp@mac.com>
+ @created 19.10.2009
+
+ 1. Poll folder on FTP server A for changes
+ 2. Copy changed files to local machine
+ 3. Upload changed files to FTP server B
+ - What if the changed file is just being uploaded to server A? (must use separate folder? Maybe the
+ server is smart enough not to list an incomplete file?)
+ - What if the poller runs before the last upload finishes? (fixed with PID checking)
+ - Does "LIST -T" work on any FTP server? (no, but we don't need it)
+ - What does "changes on server A" mean? (aything not in the local skiplist)
+ - What if the new item is a folder? (will throw exception)
+ - What if there are more than 1 new file? (it's OK)
+"""
+import yaml, pickle, os, sys, re
+from ftplib import FTP
+
+tmp_path = ""
+sl_path = ".skiplist"
+pfile_path = "/private/tmp/MediaRelay.pid"
+
+# Borrowed from http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
+try:
+ pf = file(pfile_path, "r")
+ pid = int(pf.read().strip())
+ pf.close()
+except Exception, e:
+ pid = None
+
+if pid:
+ sys.stderr.write("Relay already running with PID %d\n" % (pid))
+ sys.exit(1)
+
+try:
+ c = yaml.load(file("config.yaml", "r"))
+except Exception, e:
+ sys.stderr.write("Cannot load config file")
+ sys.exit(1)
+
+# Remember who we are just in case this gets called during transfer
+pid = str(os.getpid())
+pf = file(pfile_path, "w")
+pf.write(pid)
+pf.close()
+
+# The global skiplist stores the listings of all configs
+g_skiplist = {}
+
+# Loop through the config
+for l in c:
+ for k in l:
+ (src_auth, src_url) = re.split("@", l[k]['source'])
+ (src_uname, src_pwd) = re.split(":", src_auth)
+ (src_host, src_path) = re.split("/", src_url, maxsplit=1)
+ (dst_auth, dst_url) = re.split("@", l[k]['destination'])
+ (dst_uname, dst_pwd) = re.split(":", dst_auth)
+ (dst_host, dst_path) = re.split("/", dst_url, maxsplit=1)
+
+ ftp_source = FTP(src_host)
+ ftp_source.login(src_uname, src_pwd)
+
+ ftp_destination = FTP(dst_host)
+ ftp_destination.login(dst_uname, src_pwd)
+
+ new = []; srv_list = []; skiplist = []
+
+ # Load skiplist from file
+ try:
+ skiplist = pickle.load(file(sl_path, "r"))
+ except Exception, e:
+ skiplist = {k:[]}
+ g_skiplist[k] = []
+
+ # Load current list from source server
+ srv_list = ftp_source.nlst(src_path)
+ # Filter out skiplist
+ new = filter(lambda i: i not in skiplist[k], srv_list)
+
+ print "-- checking %s" % (k)
+
+ # Download new files
+ for f in new:
+ try:
+ print " <- getting %s" % (f)
+ ftp_cmd = "RETR %s/%s" % (src_path, f)
+ ftp_source.retrbinary(ftp_cmd, open(f, "wb").write)
+ except Exception, e:
+ sys.stderr.write("Error getting %s - %s\n" % (f, e))
+ else:
+ try:
+ # Upload new file to destination
+ print " -> sending %s" % (f)
+ ftp_cmd = "STOR %s/%s" % (dst_path, f)
+ ftp_destination.storbinary(ftp_cmd, open(f, "r"))
+ except Exception, e:
+ sys.stderr.write("Error sending %s - %s\n" % (f, e))
+
+ os.remove(f)
+
+ if len(new) < 1:
+ print " -- nothing to download"
+
+ # Update global skiplist
+ g_skiplist[k] = srv_list
+
+ # Close FTP connections
+ ftp_source.close(); ftp_destination.close()
+
+# Update skiplist
+pickle.dump(g_skiplist, file(sl_path, "w"))
+
+# Remove PID file
+os.remove(pfile_path)