git-ftp: retry, sftp support
- From
Martin Langhoff <martin.langhoff@gmail.com>
- Date
- Jul 18, 2011, 02:35 UTC
- Message-ID
- <CACPiFCL22yr096_nNfjvfP_bkJRC7HA65GUF12wedzV=cz-_kg@mail.gmail.com>
René, Timo,
Thanks for git-ftp -- it has saved me from going crazy with low cost hosting setups that only support ftp.
Along the way, I scratched some personal itches; so here are the patches.
One thing I have not been able to fix: it uses one connection per file. This creates significant overhead for large trees, and in some cases it triggers firewalls' DoS countermeasures. An initial attempt at fixing it using ncftpspooler did not pan out. It will probably require rewriting the script in Python or Perl to use libcurl bindings.
[ Unfortunately, I am behind the great fw of china right now, and I can't seem to smtp out via gmail from here. So patches attached. ]
cheers,
martin
--
martin.langhoff@gmail.com
martin@laptop.org -- Software Architect - OLPC
- ask interesting questions
- don't get distracted with shiny stuff - working code first
- http://wiki.laptop.org/go/User:Martinlanghoff
From e193aa15ae0b59296a04dc9910d36c7d9059ff3f Mon Sep 17 00:00:00 2001
From: Martin Langhoff <martin@laptop.org>
Date: Sun, 10 Jul 2011 11:14:50 -0400
Subject: [PATCH 1/2] Add support for --retry parameter -- which gets passed
along to curl
Makes a difference with unreliable connections and hosts.
---
git-ftp.sh | 27 ++++++++++++++++++++++++---
1 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/git-ftp.sh b/git-ftp.sh
index cfadc5f..5dda3a4 100755
--- a/git-ftp.sh
+++ b/git-ftp.sh
@@ -58,6 +58,7 @@ OPTIONS:
-D, --dry-run Dry run: Does not upload anything
-a, --all Uploads all files, ignores deployed SHA1 hash
-c, --catchup Updates SHA1 hash without uploading files
+ -r, --retry Number of retries (see curl manpage)
-f, --force Force, does not ask questions
-v, --verbose Verbose
@@ -123,17 +124,17 @@ upload_file() {
if [ -z ${dest_file} ]; then
dest_file=${source_file}
fi
- ${CURL_BIN} -T ${source_file} --user ${REMOTE_USER}:${REMOTE_PASSWD} --ftp-create-dirs -# ftp://${REMOTE_HOST}/${REMOTE_PATH}${dest_file}
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -T ${source_file} --user ${REMOTE_USER}:${REMOTE_PASSWD} --ftp-create-dirs -# ftp://${REMOTE_HOST}/${REMOTE_PATH}${dest_file}
}
remove_file() {
file=${1}
- ${CURL_BIN} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} -Q "-DELE ${REMOTE_PATH}${file}" ftp://${REMOTE_HOST} > /dev/null 2>&1
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} -Q "-DELE ${REMOTE_PATH}${file}" ftp://${REMOTE_HOST} > /dev/null 2>&1
}
get_file_content() {
source_file=${1}
- ${CURL_BIN} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} ftp://${REMOTE_HOST}/${REMOTE_PATH}${source_file}
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} ftp://${REMOTE_HOST}/${REMOTE_PATH}${source_file}
}
while test $# != 0
@@ -176,6 +177,22 @@ do
;;
esac
;;
+ -r|--retry*)
+ case "$#,$1" in
+ *,*=*)
+ CURL_RETRY_INT=`expr "z$1" : 'z-[^=]*=\(.*\)'`
+ ;;
+ 1,*)
+ usage
+ ;;
+ *)
+ if [ ! `echo "${2}" | egrep '^-' | wc -l` -eq 1 ]; then
+ CURL_RETRY_INT="$2"
+ shift
+ fi
+ ;;
+ esac
+ ;;
-a|--all)
IGNORE_DEPLOYED=1
;;
@@ -309,6 +326,10 @@ write_log "Host is '${REMOTE_HOST}'"
write_log "User is '${REMOTE_USER}'"
write_log "Path is '${REMOTE_PATH}'"
+if [ -n "$CURL_RETRY_INT" ];then
+ CURL_RETRY='--retry'
+fi
+
DEPLOYED_SHA1=""
if [ ${IGNORE_DEPLOYED} -ne 1 ]; then
# Get the last commit (SHA) we deployed if not ignored or not found
--
1.7.6
From c2bbf06d43bfb4fd0df461f09ccd0dd895282d04 Mon Sep 17 00:00:00 2001
From: Martin Langhoff <martin@laptop.org>
Date: Sun, 10 Jul 2011 12:57:49 -0400
Subject: [PATCH 2/2] Add sftp support
curl already does the heavy lifting. This is non-ideal
in that it passes the password around in the cmdline.
This is shell however -- almost anything we do with a password means
passing params around in horrifying ways.
Pending: a way to use ssh keys.
---
git-ftp.sh | 17 +++++++++--------
1 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/git-ftp.sh b/git-ftp.sh
index 5dda3a4..9c57e4d 100755
--- a/git-ftp.sh
+++ b/git-ftp.sh
@@ -49,7 +49,7 @@ DESCRIPTION:
URL:
. default host.example.com[:<port>][/<remote path>]
- . FTP ftp://host.example.com[:<port>][/<remote path>]
+ . FTP (s)ftp://host.example.com[:<port>][/<remote path>]
OPTIONS:
-h, --help Show this message
@@ -65,6 +65,7 @@ OPTIONS:
EXAMPLES:
. git ftp -u john ftp://ftp.example.com:4445/public_ftp -p -v
. git ftp -p -u john -v ftp.example.com:4445:/public_ftp
+ . git ftp -u john sftp://ftp.secureexample.com/public_ftp -p -v
EOF
exit 0
}
@@ -124,17 +125,17 @@ upload_file() {
if [ -z ${dest_file} ]; then
dest_file=${source_file}
fi
- ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -T ${source_file} --user ${REMOTE_USER}:${REMOTE_PASSWD} --ftp-create-dirs -# ftp://${REMOTE_HOST}/${REMOTE_PATH}${dest_file}
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -T ${source_file} --user ${REMOTE_USER}:${REMOTE_PASSWD} --ftp-create-dirs -# ${REMOTE_PROTOCOL}://${REMOTE_HOST}/${REMOTE_PATH}${dest_file}
}
remove_file() {
file=${1}
- ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} -Q "-DELE ${REMOTE_PATH}${file}" ftp://${REMOTE_HOST} > /dev/null 2>&1
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} -Q "-DELE ${REMOTE_PATH}${file}" ${REMOTE_PROTOCOL}://${REMOTE_HOST} > /dev/null 2>&1
}
get_file_content() {
source_file=${1}
- ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} ftp://${REMOTE_HOST}/${REMOTE_PATH}${source_file}
+ ${CURL_BIN} ${CURL_RETRY} ${CURL_RETRY_INT} -s --user ${REMOTE_USER}:${REMOTE_PASSWD} ${REMOTE_PROTOCOL}://${REMOTE_HOST}/${REMOTE_PATH}${source_file}
}
while test $# != 0
@@ -305,7 +306,7 @@ if [ ${HAS_ERROR} -ne 0 ]; then
fi
# Split protocol from url
-REMOTE_PROTOCOL=`expr "${URL}" : "\(ftp\).*"`
+REMOTE_PROTOCOL=`expr "${URL}" : "\(sftp\|ftp\).*"`
# Check supported protocol
if [ -z ${REMOTE_PROTOCOL} ]; then
@@ -333,7 +334,7 @@ fi
DEPLOYED_SHA1=""
if [ ${IGNORE_DEPLOYED} -ne 1 ]; then
# Get the last commit (SHA) we deployed if not ignored or not found
- write_log "Retrieving last commit from ftp://${REMOTE_HOST}/${REMOTE_PATH}"
+ write_log "Retrieving last commit from ${REMOTE_PROTOCOL}://${REMOTE_HOST}/${REMOTE_PATH}"
DEPLOYED_SHA1="`get_file_content ${DEPLOYED_SHA1_FILE}`"
if [ $? -ne 0 ]; then
write_info "Could not get last commit or it does not exist"
@@ -394,7 +395,7 @@ if [ $CATCHUP -ne 1 ]; then
# File exits?
if [ -f ${file} ]; then
# Uploading file
- write_info "[${done_items} of ${total_items}] Uploading ${file} to ftp://${REMOTE_HOST}/${REMOTE_PATH}${file}"
+ write_info "[${done_items} of ${total_items}] Uploading ${file} to ${REMOTE_PROTOCOL}://${REMOTE_HOST}/${REMOTE_PATH}${file}"
if [ ${DRY_RUN} -ne 1 ]; then
upload_file ${file}
check_exit_status "Could not upload"
@@ -414,7 +415,7 @@ fi
# if successful, remember the SHA1 of last commit
DEPLOYED_SHA1=`${GIT_BIN} log -n 1 --pretty=%H`
-write_info "Uploading commit log to ftp://${REMOTE_HOST}/${REMOTE_PATH}${DEPLOYED_SHA1_FILE}"
+write_info "Uploading commit log to ${REMOTE_PROTOCOL}://${REMOTE_HOST}/${REMOTE_PATH}${DEPLOYED_SHA1_FILE}"
if [ ${DRY_RUN} -ne 1 ]; then
echo "${DEPLOYED_SHA1}" | upload_file - ${DEPLOYED_SHA1_FILE}
check_exit_status "Could not upload"
--
1.7.6