Merge branch 'joeyconfig'

8 files changed, 203 insertions, 90 deletions

diff --git a/src/Propellor/Bootstrap.hs b/src/Propellor/Bootstrap.hs
new file mode 100644
index 00000000..45340832
--- /dev/null
+++ b/src/Propellor/Bootstrap.hs
@@ -0,0 +1,97 @@
+module Propellor.Bootstrap (
+	bootstrapPropellorCommand,
+	installGitCommand,
+	buildPropellor,
+) where
+
+import Propellor
+import Utility.SafeCommand
+
+import System.Posix.Files
+import Data.List
+
+type ShellCommand = String
+
+-- Shell command line to build propellor, used when bootstrapping on a new
+-- host. Should be run inside the propellor source tree, and will install
+-- all necessary build dependencies.
+bootstrapPropellorCommand :: ShellCommand
+bootstrapPropellorCommand = "if ! test -x ./propellor; then " ++ go ++ "; fi"
+  where
+	go = intercalate " && "
+		[ depsCommand
+		, buildCommand
+		]
+
+buildCommand :: ShellCommand
+buildCommand = intercalate " && "
+	[ "cabal configure"
+	, "cabal build"
+	, "ln -sf dist/build/propellor-config/propellor-config propellor"
+	]
+
+depsCommand :: ShellCommand
+depsCommand = 
+	"(" ++ aptinstall debdeps ++ " || (apt-get update && " ++ aptinstall debdeps ++ ")) && "
+	++ "(" ++ aptinstall ["libghc-async-dev"] ++ " || (" ++ cabalinstall ["async"] ++  ")) || "
+	++ "(" ++ cabalinstall ["--only-dependencies"] ++ ")"
+  where
+	aptinstall ps = "apt-get --no-upgrade --no-install-recommends -y install " ++ unwords ps
+
+	cabalinstall ps = "cabal update; cabal install " ++ unwords ps
+
+	-- This is the same build deps listed in debian/control.
+	debdeps =
+		[ "gnupg"
+		, "ghc"
+		, "cabal-install"
+		-- async is not available in debian stable
+		-- , "libghc-async-dev"
+		, "libghc-missingh-dev"
+		, "libghc-hslogger-dev"
+		, "libghc-unix-compat-dev"
+		, "libghc-ansi-terminal-dev"
+		, "libghc-ifelse-dev"
+		, "libghc-network-dev"
+		, "libghc-quickcheck2-dev"
+		, "libghc-mtl-dev"
+		, "libghc-monadcatchio-transformers-dev"
+		]
+
+
+installGitCommand :: ShellCommand
+installGitCommand = "if ! git --version >/dev/null; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git; fi"
+
+buildPropellor :: IO ()
+buildPropellor = unlessM (actionMessage "Propellor build" build) $
+	errorMessage "Propellor build failed!"
+
+-- Build propellor using cabal, and symlink propellor to where cabal
+-- leaves the built binary.
+--
+-- For speed, only runs cabal configure when it's not been run before.
+-- If the build fails cabal may need to have configure re-run.
+build :: IO Bool
+build = catchBoolIO $ do
+	make "dist/setup-config" ["propellor.cabal"] $
+		cabal ["configure"]
+	unlessM (cabal ["build"]) $ do
+		void $ cabal ["configure"]
+		unlessM (cabal ["build"]) $
+			error "cabal build failed"
+	nukeFile "propellor"
+	createSymbolicLink "dist/build/propellor-config/propellor-config" "propellor"
+	return True
+
+make :: FilePath -> [FilePath] -> IO Bool -> IO ()
+make dest srcs builder = do
+	dt <- getmtime dest
+	st <- mapM getmtime srcs
+	when (dt == Nothing || any (> dt) st) $
+		unlessM builder $
+			error $ "failed to make " ++ dest
+  where
+	getmtime = catchMaybeIO . getModificationTime
+
+cabal :: [String] -> IO Bool
+cabal = boolSystem "cabal" . map Param
diff --git a/src/Propellor/CmdLine.hs b/src/Propellor/CmdLine.hs
index 15dc09c3..9d7d0d95 100644
--- a/src/Propellor/CmdLine.hs
+++ b/src/Propellor/CmdLine.hs
@@ -12,6 +12,7 @@ import qualified Network.BSD
 import Propellor
 import Propellor.Gpg
 import Propellor.Git
+import Propellor.Bootstrap
 import Propellor.Spin
 import Propellor.Types.CmdLine
 import qualified Propellor.Property.Docker as Docker
@@ -31,6 +32,7 @@ usage h = hPutStrLn h $ unlines
 	, "  propellor --edit field context"
 	, "  propellor --list-fields"
 	, "  propellor --merge"
+	, "  propellor --build"
 	]
 
 usageError :: [String] -> IO a
@@ -128,19 +130,16 @@ unknownhost h hosts = errorMessage $ unlines
 	]
 
 buildFirst :: CmdLine -> IO () -> IO ()
-buildFirst cmdline next = ifM (doesFileExist "Makefile")
-	( do
-		oldtime <- getmtime
-		ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
-			( do
-				newtime <- getmtime
-				if newtime == oldtime
-					then next
-					else void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
-			, errorMessage "Propellor build failed!" 
-			)
-	, next
-	)
+buildFirst cmdline next = do
+	oldtime <- getmtime
+	buildPropellor
+	newtime <- getmtime
+	if newtime == oldtime
+		then next
+		else void $ boolSystem "./propellor"
+			[ Param "--continue"
+			, Param (show cmdline)
+			]
   where
 	getmtime = catchMaybeIO $ getModificationTime "propellor"
 
@@ -155,10 +154,12 @@ updateFirst cmdline next = ifM hasOrigin (updateFirst' cmdline next, next)
 
 updateFirst' :: CmdLine -> IO () -> IO ()
 updateFirst' cmdline next = ifM fetchOrigin
-	( ifM (actionMessage "Propellor build" $ boolSystem "make" [Param "build"])
-		( void $ boolSystem "./propellor" [Param "--continue", Param (show cmdline)]
-			, errorMessage "Propellor build failed!" 
-		)
+	( do
+		buildPropellor
+		void $ boolSystem "./propellor"
+			[ Param "--continue"
+			, Param (show cmdline)
+			]
 	, next
 	)
 
diff --git a/src/Propellor/Property/Cron.hs b/src/Propellor/Property/Cron.hs
index fd365c8f..2a28a157 100644
--- a/src/Propellor/Property/Cron.hs
+++ b/src/Propellor/Property/Cron.hs
@@ -30,7 +30,10 @@ data Times
 job :: Desc -> Times -> UserName -> FilePath -> String -> Property NoInfo
 job desc times user cddir command = combineProperties ("cronned " ++ desc)
 	[ cronjobfile `File.hasContent`
-		[ "# Generated by propellor"
+		[ case times of
+			Times _ -> ""
+			_ -> "#!/bin/sh\nset -e"
+		, "# Generated by propellor"
 		, ""
 		, "SHELL=/bin/sh"
 		, "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
diff --git a/src/Propellor/Property/SiteSpecific/JoeySites.hs b/src/Propellor/Property/SiteSpecific/JoeySites.hs
index 9644cb72..005f12d1 100644
--- a/src/Propellor/Property/SiteSpecific/JoeySites.hs
+++ b/src/Propellor/Property/SiteSpecific/JoeySites.hs
@@ -125,17 +125,6 @@ mumbleServer hosts = combineProperties hn $ props
   where
 	hn = "mumble.debian.net"
 
-obnamLowMem :: Property NoInfo
-obnamLowMem = combineProperties "obnam tuned for low memory use"
-	[ Obnam.latestVersion
-	, "/etc/obnam.conf" `File.containsLines`
-		[ "[config]"
-		, "# Suggested by liw to keep Obnam memory consumption down (at some speed cost)."
-		, "upload-queue-size = 96"
-		, "lru-size = 96"
-		]
-	]
-
 -- git.kitenet.net and git.joeyh.name
 gitServer :: [Host] -> Property HasInfo
 gitServer hosts = propertyList "git.kitenet.net setup" $ props
@@ -282,7 +271,8 @@ gitAnnexDistributor = combineProperties "git-annex distributor, including rsync
 	& "/etc/default/rsync" `File.containsLine` "RSYNC_ENABLE=true"
 		`onChange` Service.running "rsync"
 	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild"
-	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-mavericks"
+	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite"
+	& endpoint "/srv/web/downloads.kitenet.net/git-annex/autobuild/windows"
 	-- git-annex distribution signing key
 	& Gpg.keyImported (Gpg.GpgKeyId "89C809CB") "joey"
   where
diff --git a/src/Propellor/Property/Tor.hs b/src/Propellor/Property/Tor.hs
index 8176e643..7a490824 100644
--- a/src/Propellor/Property/Tor.hs
+++ b/src/Propellor/Property/Tor.hs
@@ -5,9 +5,11 @@ import qualified Propellor.Property.File as File
 import qualified Propellor.Property.Apt as Apt
 import qualified Propellor.Property.Service as Service
 import Utility.FileMode
+import Utility.DataUnits
 
 import System.Posix.Files
 import Data.Char
+import Data.List
 
 type HiddenServiceName = String
 
@@ -17,60 +19,36 @@ type NodeName = String
 --
 -- Uses port 443
 isBridge :: Property NoInfo
-isBridge = isBridge' []
-
-isBridge' :: [String] -> Property NoInfo
-isBridge' extraconfig = server config
+isBridge = configured
+	[ ("BridgeRelay", "1")
+	, ("Exitpolicy", "reject *:*")
+	, ("ORPort", "443")
+	]
 	`describe` "tor bridge"
-  where
-	config = 
-		[ "BridgeRelay 1"
-		, "Exitpolicy reject *:*"
-		, "ORPort 443"
-		] ++ extraconfig
+	`requires` server
 
 -- | Sets up a tor relay.
 --
 -- Uses port 443
 isRelay :: Property NoInfo
-isRelay = isRelay' []
-
-isRelay' :: [String] -> Property NoInfo
-isRelay' extraconfig = server config
+isRelay = configured
+	[ ("BridgeRelay", "0")
+	, ("Exitpolicy", "reject *:*")
+	, ("ORPort", "443")
+	]
 	`describe` "tor relay"
-  where
-	config = 
-		[ "BridgeRelay 0"
-		, "Exitpolicy reject *:*"
-		, "ORPort 443"
-		] ++ extraconfig
+	`requires` server
 
--- | Converts a property like isBridge' or isRelay' to be a named
--- node, with a known private key.
+-- | Makes the tor node be named, with a known private key.
 --
 -- This can be moved to a different IP without needing to wait to
 -- accumulate trust.
---
--- The base property can be used to start out and then upgraded to 
--- a named property later.
-named :: NodeName -> ([String] -> Property NoInfo) -> Property HasInfo
-named n basep = p `describe` (getDesc p ++ " " ++ n)
-  where
-	p = basep ["Nickname " ++ saneNickname n]
-		`requires` torPrivKey (Context ("tor " ++ n))
-
--- | A tor server (bridge, relay, or exit)
--- Don't use if you just want to run tor for personal use.
-server :: [String] -> Property NoInfo
-server extraconfig = setup
-	`requires` Apt.installed ["tor", "ntp"]
-	`describe` "tor server"
+named :: NodeName -> Property HasInfo
+named n = configured [("Nickname", n')]
+	`describe` ("tor node named " ++ n')
+	`requires` torPrivKey (Context ("tor " ++ n))
   where
-	setup = mainConfig `File.hasContent` config
-		`onChange` restarted
-	config = 
-		[ "SocksPort 0"
-		] ++ extraconfig
+	n' = saneNickname n
 
 torPrivKey :: Context -> Property HasInfo
 torPrivKey context = f `File.hasPrivContent` context
@@ -80,15 +58,58 @@ torPrivKey context = f `File.hasPrivContent` context
   where
 	f = "/var/lib/tor/keys/secret_id_key"
 
+-- | A tor server (bridge, relay, or exit)
+-- Don't use if you just want to run tor for personal use.
+server :: Property NoInfo
+server = configured [("SocksPort", "0")]
+	`requires` Apt.installed ["tor", "ntp"]
+	`describe` "tor server"
+
+-- | Specifies configuration settings. Any lines in the config file
+-- that set other values for the specified settings will be removed,
+-- while other settings are left as-is. Tor is restarted when
+-- configuration is changed.
+configured :: [(String, String)] -> Property NoInfo
+configured settings = File.fileProperty "tor configured" go mainConfig
+	`onChange` restarted
+  where
+	ks = map fst settings
+	go ls = sort $ map toconfig $
+		filter (\(k, _) -> k `notElem` ks) (map fromconfig ls)
+		++ settings
+	toconfig (k, v) = k ++ " " ++ v
+	fromconfig = separate (== ' ')
+
+data BwLimit
+	= PerSecond String
+	| PerDay String
+	| PerMonth String
+
+-- | Limit incoming and outgoing traffic to the specified
+-- amount each.
+--
+-- For example, PerSecond "30 kibibytes" is the minimum limit
+-- for a useful relay.
+bandwidthRate :: BwLimit -> Property NoInfo
+bandwidthRate (PerSecond s) = bandwidthRate' s 1
+bandwidthRate (PerDay s) = bandwidthRate' s (24*60*60)
+bandwidthRate (PerMonth s) = bandwidthRate' s (31*24*60*60)
+
+bandwidthRate' :: String -> Integer -> Property NoInfo
+bandwidthRate' s divby = case readSize dataUnits s of
+	Just sz -> let v = show (sz `div` divby) ++ " bytes"
+		in configured [("BandwidthRate", v)]
+			`describe` ("tor BandwidthRate " ++ v)
+	Nothing -> property ("unable to parse " ++ s) noChange
+
 hiddenServiceAvailable :: HiddenServiceName -> Int -> Property NoInfo
 hiddenServiceAvailable hn port = hiddenServiceHostName prop
   where
-	prop = mainConfig `File.containsLines`
-		[ unwords ["HiddenServiceDir", varLib </> hn]
-		, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
+	prop = configured
+		[ ("HiddenServiceDir", varLib </> hn)
+		, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
 		]
 		`describe` "hidden service available"
-		`onChange` Service.reloaded "tor"
 	hiddenServiceHostName p =  adjustPropertySatisfy p $ \satisfy -> do
 		r <- satisfy
 		h <- liftIO $ readFile (varLib </> hn </> "hostname")
@@ -96,12 +117,11 @@ hiddenServiceAvailable hn port = hiddenServiceHostName prop
 		return r
 
 hiddenService :: HiddenServiceName -> Int -> Property NoInfo
-hiddenService hn port = mainConfig `File.containsLines`
-	[ unwords ["HiddenServiceDir", varLib </> hn]
-	, unwords ["HiddenServicePort", show port, "127.0.0.1:" ++ show port]
+hiddenService hn port = configured
+	[ ("HiddenServiceDir", varLib </> hn)
+	, ("HiddenServicePort", unwords [show port, "127.0.0.1:" ++ show port])
 	]
 	`describe` unwords ["hidden service available:", hn, show port]
-	`onChange` restarted
 
 hiddenServiceData :: IsContext c => HiddenServiceName -> c -> Property HasInfo
 hiddenServiceData hn context = combineProperties desc
diff --git a/src/Propellor/Shim.hs b/src/Propellor/Shim.hs
index da4c96eb..e1ea2825 100644
--- a/src/Propellor/Shim.hs
+++ b/src/Propellor/Shim.hs
@@ -33,6 +33,9 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
 	let linker = (dest ++) $ 
 		fromMaybe (error "cannot find ld-linux linker") $
 			headMaybe $ filter ("ld-linux" `isInfixOf`) libs'
+	let linkersym = takeDirectory linker </> takeFileName propellorbin
+	createSymbolicLink linkersym (takeFileName linker)
+
 	let gconvdir = (dest ++) $ takeDirectory $
 		fromMaybe (error "cannot find gconv directory") $
 			headMaybe $ filter ("/gconv/" `isInfixOf`) glibclibs
@@ -42,7 +45,7 @@ setup propellorbin propellorbinpath dest = checkAlreadyShimmed propellorbin $ do
 		[ shebang
 		, "GCONV_PATH=" ++ shellEscape gconvdir
 		, "export GCONV_PATH"
-		, "exec " ++ unwords (map shellEscape $ linker : linkerparams) ++ 
+		, "exec " ++ unwords (map shellEscape $ linkersym : linkerparams) ++ 
 			" " ++ shellEscape (fromMaybe propellorbin propellorbinpath) ++ " \"$@\""
 		]
 	modifyFileMode shim (addModes executeModes)
diff --git a/src/Propellor/Spin.hs b/src/Propellor/Spin.hs
index 5063145e..f55f2977 100644
--- a/src/Propellor/Spin.hs
+++ b/src/Propellor/Spin.hs
@@ -24,6 +24,7 @@ import Propellor.PrivData.Paths
 import Propellor.Git
 import Propellor.Ssh
 import Propellor.Gpg
+import Propellor.Bootstrap
 import Propellor.Types.CmdLine
 import qualified Propellor.Shim as Shim
 import Utility.FileMode
@@ -69,7 +70,7 @@ spin target relay hst = do
 	probecmd = intercalate " ; "
 		[ "if [ ! -d " ++ localdir ++ "/.git ]"
 		, "then (" ++ intercalate " && "
-			[ "if ! git --version || ! make --version; then apt-get update && apt-get --no-install-recommends --no-upgrade -y install git make; fi"
+			[ installGitCommand
 			, "echo " ++ toMarked statusMarker (show NeedGitClone)
 			] ++ ") || echo " ++ toMarked statusMarker (show NeedPrecompiled)
 		, "else " ++ updatecmd
@@ -78,7 +79,7 @@ spin target relay hst = do
 	
 	updatecmd = intercalate " && "
 		[ "cd " ++ localdir
-		, "if ! test -x ./propellor; then make deps build; fi"
+		, bootstrapPropellorCommand
 		, if viarelay
 			then "./propellor --continue " ++
 				shellEscape (show (Relay target))
diff --git a/src/wrapper.hs b/src/wrapper.hs
index 304e833d..034eb2bf 100644
--- a/src/wrapper.hs
+++ b/src/wrapper.hs
@@ -15,6 +15,7 @@
 module Main where
 
 import Propellor.Message
+import Propellor.Bootstrap
 import Utility.UserInfo
 import Utility.Monad
 import Utility.Process
@@ -91,13 +92,10 @@ wrapper args propellordir propellorbin = do
 					warnoutofdate propellordir True
 	buildruncfg = do
 		changeWorkingDirectory propellordir
-		ifM (boolSystem "make" [Param "build"])
-			( do
-				putStrLn ""
-				putStrLn ""
-				chain
-			, error "Propellor build failed."
-			)
+		buildPropellor
+		putStrLn ""
+		putStrLn ""
+		chain
 	chain = do
 		(_, _, _, pid) <- createProcess (proc propellorbin args) 
 		exitWith =<< waitForProcess pid