From f62d2fb18389947ce11021ba80b2aee52c6d03c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:22:19 -0400 Subject: propellor --spin can now deploy propellor to hosts that do not have git, ghc, or apt-get. This is accomplished by uploading a fairly portable precompiled tarball of propellor. --- src/Propellor/Server.hs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 513a81f4..786d1211 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -16,6 +16,7 @@ import Propellor.Protocol import Propellor.PrivData.Paths import Propellor.Git import Propellor.Ssh +import qualified Propellor.Shim as Shim import Utility.FileMode import Utility.SafeCommand @@ -69,6 +70,11 @@ updateServer hn hst connect = connect go hClose fromh sendGitClone hn updateServer hn hst connect + (Just NeedPrecompiled) -> do + hClose toh + hClose fromh + sendPrecompiled hn + updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () @@ -113,6 +119,32 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do , "rm -f " ++ remotebundle ] +-- Send a tarball containing the precompiled propellor, and libraries. +-- This should be reasonably portable, as long as the remote host has the +-- same architecture as the build host. +sendPrecompiled :: HostName -> IO () +sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort " ++ hn) $ do + cacheparams <- sshCachingParams hn + withTmpDir "propellor" $ \tmpdir -> + bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do + changeWorkingDirectory tmpdir + let shimdir = "propellor" + let me = localdir "propellor" + void $ Shim.setup me shimdir + withTmpFile "propellor.tar" $ \tarball -> allM id + [ boolSystem "strip" [File me] + , boolSystem "tar" [Param "cf", File tmp, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball) + , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] + ] + where + remotetarball = "/usr/local/propellor.tar" + unpackcmd = shellSwap $ intercalate " && " + [ "cd " ++ takeDirectory remotetarball + , "tar xf " ++ remotetarball + , "rm -f " ++ remotetarball + ] + -- Shim for git push over the propellor ssh channel. -- Reads from stdin and sends it to hout; -- reads from hin and sends it to stdout. -- cgit v1.3-2-g0d8e From 7fd338f96f5852ea309bd4a2964bf93d06062086 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:25:00 -0400 Subject: propellor spin --- src/Propellor/CmdLine.hs | 2 +- src/Propellor/Server.hs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs index ec2ca7ed..99826c3a 100644 --- a/src/Propellor/CmdLine.hs +++ b/src/Propellor/CmdLine.hs @@ -176,7 +176,7 @@ spin hn hst = do updatecmd = mkcmd [ "if [ ! -d " ++ localdir ++ " ]" , "then (" ++ intercalate " && " - [ "apt-get update" + [ "apt-get-fail update" , "apt-get --no-install-recommends --no-upgrade -y install git make" , "echo " ++ toMarked statusMarker (show NeedGitClone) ] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled) diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 786d1211..a8cd6a00 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -8,7 +8,9 @@ import Data.List import System.Exit import System.PosixCompat import System.Posix.IO +import System.Posix.Directory import Control.Concurrent.Async +import Control.Exception (bracket) import qualified Data.ByteString as B import Propellor @@ -131,15 +133,15 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor let shimdir = "propellor" let me = localdir "propellor" void $ Shim.setup me shimdir - withTmpFile "propellor.tar" $ \tarball -> allM id + withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tmp, File shimdir] - , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball) + , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] ] where remotetarball = "/usr/local/propellor.tar" - unpackcmd = shellSwap $ intercalate " && " + unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball , "tar xf " ++ remotetarball , "rm -f " ++ remotetarball -- cgit v1.3-2-g0d8e From 99e0a5fea733fbeb18ee8fa4556994a50e470b7b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:27:35 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index a8cd6a00..eff56997 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -131,7 +131,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do changeWorkingDirectory tmpdir let shimdir = "propellor" - let me = localdir "propellor" + me <- readSymbolicLink "/proc/self/exe" void $ Shim.setup me shimdir withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] -- cgit v1.3-2-g0d8e From e5135c19489ee0799f9408c2336ebf98c6ddadf3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:32:04 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index eff56997..d6987d2d 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -76,7 +76,6 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn - updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () @@ -125,14 +124,16 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do -- This should be reasonably portable, as long as the remote host has the -- same architecture as the build host. sendPrecompiled :: HostName -> IO () -sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort " ++ hn) $ do +sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort") $ do cacheparams <- sshCachingParams hn withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do changeWorkingDirectory tmpdir let shimdir = "propellor" me <- readSymbolicLink "/proc/self/exe" - void $ Shim.setup me shimdir + shim <- Shim.setup me shimdir + when (shim /= shimdir "propellor") $ + renameFile shim (shimdir "propellor") withTmpFile "propellor.tar" $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] -- cgit v1.3-2-g0d8e From f1fdd6f678d2d5d753407434a53cc6e2876185ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:34:34 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index d6987d2d..a72c83ae 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -128,13 +128,14 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor cacheparams <- sshCachingParams hn withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do - changeWorkingDirectory tmpdir let shimdir = "propellor" + changeWorkingDirectory shimdir me <- readSymbolicLink "/proc/self/exe" - shim <- Shim.setup me shimdir + shim <- Shim.setup me "." + changeWorkingDirectory tmpdir when (shim /= shimdir "propellor") $ renameFile shim (shimdir "propellor") - withTmpFile "propellor.tar" $ \tarball _ -> allM id + withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] -- cgit v1.3-2-g0d8e From 1af398d33eee40a13d7c310e6f97b170b47d21ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:35:14 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 1 + 1 file changed, 1 insertion(+) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index a72c83ae..fb37ff1a 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -129,6 +129,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do let shimdir = "propellor" + createDirectoryIfMissing True shimdir changeWorkingDirectory shimdir me <- readSymbolicLink "/proc/self/exe" shim <- Shim.setup me "." -- cgit v1.3-2-g0d8e From 083b82d8100ef1915a00f8fbcd1530b85448dc9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:36:16 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index fb37ff1a..fde9c2ec 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -129,13 +129,13 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor withTmpDir "propellor" $ \tmpdir -> bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do let shimdir = "propellor" - createDirectoryIfMissing True shimdir - changeWorkingDirectory shimdir + createDirectoryIfMissing True (tmpdir shimdir) + changeWorkingDirectory (tmpdir shimdir) me <- readSymbolicLink "/proc/self/exe" shim <- Shim.setup me "." + when (shim /= "propellor") $ + renameFile shim "propellor" changeWorkingDirectory tmpdir - when (shim /= shimdir "propellor") $ - renameFile shim (shimdir "propellor") withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] , boolSystem "tar" [Param "cf", File tarball, File shimdir] -- cgit v1.3-2-g0d8e From 0d65fdeb1dcb414b0fb1d5719dd734a1b04da1e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:44:13 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index fde9c2ec..06fede7e 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -27,17 +27,19 @@ import Utility.SafeCommand -- running the updateServer update :: IO () update = do - req NeedRepoUrl repoUrlMarker setRepoUrl + whenM hasOrigin $ + req NeedRepoUrl repoUrlMarker setRepoUrl makePrivDataDir req NeedPrivData privDataMarker $ writeFileProtected privDataLocal - req NeedGitPush gitPushMarker $ \_ -> do - hin <- dup stdInput - hout <- dup stdOutput - hClose stdin - hClose stdout - unlessM (boolSystem "git" (pullparams hin hout)) $ - errorMessage "git pull from client failed" + whenM hasOrigin $ + req NeedGitPush gitPushMarker $ \_ -> do + hin <- dup stdInput + hout <- dup stdOutput + hClose stdin + hClose stdout + unlessM (boolSystem "git" (pullparams hin hout)) $ + errorMessage "git pull from client failed" where pullparams hin hout = [ Param "pull" @@ -76,6 +78,7 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn + loop Nothing -> return () sendRepoUrl :: Handle -> IO () -- cgit v1.3-2-g0d8e From 5d6e1a27a5a39f05b254a7fc351a2dc34ef4620c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:46:07 -0400 Subject: propellor spin --- src/Propellor/Server.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 06fede7e..eaf9649f 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -78,7 +78,7 @@ updateServer hn hst connect = connect go hClose toh hClose fromh sendPrecompiled hn - loop + updateServer hn hst connect Nothing -> return () sendRepoUrl :: Handle -> IO () -- cgit v1.3-2-g0d8e From cd6ca049b8ba90bdb4a1ba6ebf258fc68809049a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:50:56 -0400 Subject: cleanup --- src/Propellor/Server.hs | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index eaf9649f..001b4762 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -128,25 +128,28 @@ sendGitClone hn = void $ actionMessage ("Clone git repository to " ++ hn) $ do -- same architecture as the build host. sendPrecompiled :: HostName -> IO () sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor as a last resort") $ do - cacheparams <- sshCachingParams hn - withTmpDir "propellor" $ \tmpdir -> - bracket getWorkingDirectory changeWorkingDirectory $ \_ -> do - let shimdir = "propellor" - createDirectoryIfMissing True (tmpdir shimdir) - changeWorkingDirectory (tmpdir shimdir) - me <- readSymbolicLink "/proc/self/exe" - shim <- Shim.setup me "." - when (shim /= "propellor") $ - renameFile shim "propellor" - changeWorkingDirectory tmpdir - withTmpFile "propellor.tar." $ \tarball _ -> allM id - [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tarball, File shimdir] - , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] - , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] - ] + bracket getWorkingDirectory changeWorkingDirectory $ \_ -> + withTmpDir "propellor" go where + go tmpdir = do + cacheparams <- sshCachingParams hn + let shimdir = takeFileName localdir + createDirectoryIfMissing True (tmpdir shimdir) + changeWorkingDirectory (tmpdir shimdir) + me <- readSymbolicLink "/proc/self/exe" + shim <- Shim.setup me "." + when (shim /= "propellor") $ + renameFile shim "propellor" + changeWorkingDirectory tmpdir + withTmpFile "propellor.tar." $ \tarball _ -> allM id + [ boolSystem "strip" [File me] + , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] + , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] + ] + remotetarball = "/usr/local/propellor.tar" + unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball , "tar xf " ++ remotetarball -- cgit v1.3-2-g0d8e From c0662e6258e5fc8f5755f01de7d7184c532c6431 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 00:52:59 -0400 Subject: comment --- src/Propellor/Server.hs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 001b4762..743490a4 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -1,3 +1,7 @@ +-- When propellor --spin is running, the local host acts as a server, +-- which connects to the remote host's propellor and responds to its +-- requests. + module Propellor.Server ( update, updateServer, -- cgit v1.3-2-g0d8e From 128e16472a03929b089dcf44e28c651edcefcd43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 22 Nov 2014 01:13:53 -0400 Subject: compress tarball 11 mb -> 4 mb --- src/Propellor/Server.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/Propellor/Server.hs') diff --git a/src/Propellor/Server.hs b/src/Propellor/Server.hs index 743490a4..19a2c901 100644 --- a/src/Propellor/Server.hs +++ b/src/Propellor/Server.hs @@ -147,7 +147,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor changeWorkingDirectory tmpdir withTmpFile "propellor.tar." $ \tarball _ -> allM id [ boolSystem "strip" [File me] - , boolSystem "tar" [Param "cf", File tarball, File shimdir] + , boolSystem "tar" [Param "czf", File tarball, File shimdir] , boolSystem "scp" $ cacheparams ++ [File tarball, Param ("root@"++hn++":"++remotetarball)] , boolSystem "ssh" $ cacheparams ++ [Param ("root@"++hn), Param unpackcmd] ] @@ -156,7 +156,7 @@ sendPrecompiled hn = void $ actionMessage ("Uploading locally compiled propellor unpackcmd = shellWrap $ intercalate " && " [ "cd " ++ takeDirectory remotetarball - , "tar xf " ++ remotetarball + , "tar xzf " ++ remotetarball , "rm -f " ++ remotetarball ] -- cgit v1.3-2-g0d8e