From c8a653af5dfd6f876d2c5d77ef047ba47274e3d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 17:27:21 -0400 Subject: propellor spin --- config-joey.hs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 45805897..3d464e37 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -189,28 +189,37 @@ hosts = -- (o) ` (Just "remotes/origin/old-kitenet.net") -- git-annex autobuilder containers - , gitAnnexBuilder "amd64" 15 - , gitAnnexBuilder "i386" 45 - -- armel builder has a companion container that run amd64 and + , standardGitAnnexBuilder "amd64" 15 "2h" + , standardGitAnnexBuilder "i386" 45 "2h" + -- armel builder has a companion container using amd64 that -- runs the build first to get TH splices. They share a home -- directory, and need to have the same versions of all haskell - -- libraries installed. + -- libraries installed. The armel builder can ssh in to the + -- companion. , Docker.container "armel-git-annex-builder-companion" (image $ System (Debian Unstable) "amd64") & Docker.volume GitAnnexBuilder.homedir & Apt.unattendedUpgrades + & GitAnnexBuilder.treeDeps "armel" + & GitAnnexBuilder.cabalDeps + & GitAnnexBuilder.sshKeyGen + & Docker.expose "22" + & Apt.serviceInstalledRunning "ssh" , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") & Docker.link "armel-git-annex-builder-companion" "companion" & Docker.volumes_from "armel-git-annex-builder-companion" --- & GitAnnexBuilder.builder "armel" "15 * * * *" True + & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/linux/install-haskell-packages) & Apt.unattendedUpgrades ] ++ monsters -gitAnnexBuilder :: Architecture -> Int -> Host -gitAnnexBuilder arch buildminute = Docker.container (arch ++ "-git-annex-builder") +standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host +standardGitAnnexBuilder arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") (image $ System (Debian Unstable) arch) - & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") True + & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True & Apt.unattendedUpgrades -- This is my standard system setup. -- cgit v1.3-2-g0d8e From 9822babea750dc038422ac9a781d816068c3c87b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 18:43:23 -0400 Subject: propellor spin --- config-joey.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 3d464e37..9fa5dbaf 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -208,7 +208,7 @@ hosts = -- (o) ` , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") & Docker.link "armel-git-annex-builder-companion" "companion" - & Docker.volumes_from "armel-git-annex-builder-companion" + & Docker.volumes_from GitAnnexBuilder.homedir & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True -- TODO: automate installing haskell libs -- (Currently have to run -- cgit v1.3-2-g0d8e From ef3260af9542fdfbde3ac1e15098478a55d0c137 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 18:49:36 -0400 Subject: propellor spin --- config-joey.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 9fa5dbaf..9bf8f1a7 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -207,8 +207,8 @@ hosts = -- (o) ` & Apt.serviceInstalledRunning "ssh" , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") - & Docker.link "armel-git-annex-builder-companion" "companion" - & Docker.volumes_from GitAnnexBuilder.homedir + & Docker.link "armel-git-annex-builder-companion.orca.kitenet.net.propellor" "companion" + & Docker.volumes_from "armel-git-annex-builder-companion.orca.kitenet.net.propellor" & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True -- TODO: automate installing haskell libs -- (Currently have to run -- cgit v1.3-2-g0d8e From c8b59684cd45ef2736a4e492e659844ea74e2c85 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 18:53:56 -0400 Subject: propellor spin --- config-joey.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 9bf8f1a7..3d464e37 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -207,8 +207,8 @@ hosts = -- (o) ` & Apt.serviceInstalledRunning "ssh" , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") - & Docker.link "armel-git-annex-builder-companion.orca.kitenet.net.propellor" "companion" - & Docker.volumes_from "armel-git-annex-builder-companion.orca.kitenet.net.propellor" + & Docker.link "armel-git-annex-builder-companion" "companion" + & Docker.volumes_from "armel-git-annex-builder-companion" & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True -- TODO: automate installing haskell libs -- (Currently have to run -- cgit v1.3-2-g0d8e From e2c67a6e18a27faab46f5842535d87ae26f167be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 19 May 2014 19:32:16 -0400 Subject: propellor spin --- config-joey.hs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 3d464e37..5e731dc0 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -192,28 +192,33 @@ hosts = -- (o) ` , standardGitAnnexBuilder "amd64" 15 "2h" , standardGitAnnexBuilder "i386" 45 "2h" -- armel builder has a companion container using amd64 that - -- runs the build first to get TH splices. They share a home - -- directory, and need to have the same versions of all haskell - -- libraries installed. The armel builder can ssh in to the - -- companion. + -- runs the build first to get TH splices. They need + -- to have the same versions of all haskell libraries installed. , Docker.container "armel-git-annex-builder-companion" (image $ System (Debian Unstable) "amd64") - & Docker.volume GitAnnexBuilder.homedir & Apt.unattendedUpgrades + -- This volume is shared with the armel builder. + & Docker.volume GitAnnexBuilder.homedir + & File.ownerGroup GitAnnexBuilder.homedir GitAnnexBuilder.builduser GitAnnexBuilder.builduser + -- Install current versions of build deps from cabal. & GitAnnexBuilder.treeDeps "armel" & GitAnnexBuilder.cabalDeps - & GitAnnexBuilder.sshKeyGen + -- The armel builder can ssh to this companion, + -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, & Docker.expose "22" & Apt.serviceInstalledRunning "ssh" + -- ssh key is shared via the home directory volume + & GitAnnexBuilder.sshKeyGen , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") + & Apt.unattendedUpgrades + & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" & Docker.volumes_from "armel-git-annex-builder-companion" - & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages) - & Apt.unattendedUpgrades + & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True ] ++ monsters standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host -- cgit v1.3-2-g0d8e From 5ace43fb4ad8b01bbff011ee307667d52ae71eba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 13:14:13 -0400 Subject: propellor spin --- config-joey.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 5e731dc0..782c579b 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -217,7 +217,8 @@ hosts = -- (o) ` & Docker.volumes_from "armel-git-annex-builder-companion" -- TODO: automate installing haskell libs -- (Currently have to run - -- git-annex/standalone/linux/install-haskell-packages) + -- git-annex/standalone/linux/install-haskell-packages + -- which is not fully automated.) & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True ] ++ monsters -- cgit v1.3-2-g0d8e From 9acd70eb4c2eb680eed5affda4be8eb59b7507f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 18:28:13 -0400 Subject: propellor spin --- config-joey.hs | 9 +++--- .../Property/SiteSpecific/GitAnnexBuilder.hs | 33 +++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 782c579b..7f32d8b6 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - & Docker.docked hosts "armel-git-annex-builder-companion" - & Docker.docked hosts "armel-git-annex-builder" + ! Docker.docked hosts "armel-git-annex-builder-companion" + ! Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -201,7 +201,8 @@ hosts = -- (o) ` & Docker.volume GitAnnexBuilder.homedir & File.ownerGroup GitAnnexBuilder.homedir GitAnnexBuilder.builduser GitAnnexBuilder.builduser -- Install current versions of build deps from cabal. - & GitAnnexBuilder.treeDeps "armel" + & GitAnnexBuilder.tree "armel" + & GitAnnexBuilder.buildDepsNoHaskellLibs & GitAnnexBuilder.cabalDeps -- The armel builder can ssh to this companion, -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, @@ -219,7 +220,7 @@ hosts = -- (o) ` -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) - & GitAnnexBuilder.builder "armel" "1 3 * * *" "5h" True + & GitAnnexBuilder.builder' GitAnnexBuilder.buildDepsNoHaskellLibs "armel" "1 3 * * *" "5h" True ] ++ monsters standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index ddb61381..1dbdbca7 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -21,8 +21,12 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" - [ treeDeps buildarch +builder = builder' buildDeps + +builder' :: Property -> Architecture -> CronTimes -> TimeOut -> Bool -> Property +builder' buildepsprop buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" + [ tree buildarch + , buildepsprop , Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" @@ -45,13 +49,10 @@ builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbui ) ] -treeDeps :: Architecture -> Property -treeDeps buildarch = combineProperties "gitannexbuilder" - [ Apt.stdSourcesList Unstable - , Apt.buildDep ["git-annex"] - , Apt.installed ["git", "rsync", "moreutils", "ca-certificates", - "liblockfile-simple-perl", "cabal-install", "vim", "less"] - , User.accountFor builduser +tree :: Architecture -> Property +tree buildarch = combineProperties "gitannexbuilder tree" + [ User.accountFor builduser + , Apt.installed ["git"] , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir , "cd " ++ gitbuilderdir @@ -61,9 +62,21 @@ treeDeps buildarch = combineProperties "gitannexbuilder" , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser [ "git clone git://git-annex.branchable.com/ " ++ builddir ] + ] + +buildDeps :: Property +buildDeps = combineProperties "gitannexbuilder build deps" + [ Apt.stdSourcesList Unstable + , Apt.buildDep ["git-annex"] + , buildDepsNoHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir ] +buildDepsNoHaskellLibs :: Property +buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certificates", + "debhelper", "ghc", "curl", "openssh-client", "git-remote-gcrypt", + "liblockfile-simple-perl", "cabal-install", "vim", "less"] + -- Installs current versions of git-annex's deps from cabal, but only -- does so once. cabalDeps :: Property @@ -79,7 +92,7 @@ sshKeyGen = combineProperties "sshkeygen" , flagFile auth authkeys ] where - gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ f] + gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ privkey] auth = userScriptProperty builduser ["cp " ++ pubkey ++ " " ++ authkeys] privkey = homedir ".ssh" "id_rsa" pubkey = privkey ++ ".pub" -- cgit v1.3-2-g0d8e From e78d357dbe164269b5f41b26f651b7d3ae234a50 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 18:31:28 -0400 Subject: propellor spin --- config-joey.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 7f32d8b6..10e19a0b 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - ! Docker.docked hosts "armel-git-annex-builder-companion" - ! Docker.docked hosts "armel-git-annex-builder" + & Docker.docked hosts "armel-git-annex-builder-companion" + & Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily -- cgit v1.3-2-g0d8e From 8f9ab315e6460166eab17c414ddf0a581dc1250b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 18:49:03 -0400 Subject: propellor spin --- config-joey.hs | 1 - src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 10e19a0b..9972d5db 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -199,7 +199,6 @@ hosts = -- (o) ` & Apt.unattendedUpgrades -- This volume is shared with the armel builder. & Docker.volume GitAnnexBuilder.homedir - & File.ownerGroup GitAnnexBuilder.homedir GitAnnexBuilder.builduser GitAnnexBuilder.builduser -- Install current versions of build deps from cabal. & GitAnnexBuilder.tree "armel" & GitAnnexBuilder.buildDepsNoHaskellLibs diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 1dbdbca7..0cd5d3c5 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -4,6 +4,7 @@ import Propellor import qualified Propellor.Property.Apt as Apt import qualified Propellor.Property.User as User import qualified Propellor.Property.Cron as Cron +import qualified Propellor.Property.File as File import Propellor.Property.Cron (CronTimes) builduser :: UserName @@ -52,6 +53,8 @@ builder' buildepsprop buildarch crontimes timeout rsyncupload = combinePropertie tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" [ User.accountFor builduser + -- Home directory already exists when docker volume is used. + , File.ownerGroup homedir builduser builduser , Apt.installed ["git"] , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir -- cgit v1.3-2-g0d8e From eeb2d688bdf435158929bb3cbfea6fb68076de9e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 20 May 2014 19:49:07 -0400 Subject: propellor spin --- config-joey.hs | 5 ++++- src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 9972d5db..f495004e 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -196,6 +196,7 @@ hosts = -- (o) ` -- to have the same versions of all haskell libraries installed. , Docker.container "armel-git-annex-builder-companion" (image $ System (Debian Unstable) "amd64") + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades -- This volume is shared with the armel builder. & Docker.volume GitAnnexBuilder.homedir @@ -211,6 +212,7 @@ hosts = -- (o) ` & GitAnnexBuilder.sshKeyGen , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" @@ -225,8 +227,9 @@ hosts = -- (o) ` standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host standardGitAnnexBuilder arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") (image $ System (Debian Unstable) arch) - & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True + & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades + & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True -- This is my standard system setup. standardSystem :: HostName -> DebianSuite -> Architecture -> Host diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 0cd5d3c5..428b3d00 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -69,8 +69,7 @@ tree buildarch = combineProperties "gitannexbuilder tree" buildDeps :: Property buildDeps = combineProperties "gitannexbuilder build deps" - [ Apt.stdSourcesList Unstable - , Apt.buildDep ["git-annex"] + [ Apt.buildDep ["git-annex"] , buildDepsNoHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir ] @@ -78,7 +77,11 @@ buildDeps = combineProperties "gitannexbuilder build deps" buildDepsNoHaskellLibs :: Property buildDepsNoHaskellLibs = Apt.installed ["git", "rsync", "moreutils", "ca-certificates", "debhelper", "ghc", "curl", "openssh-client", "git-remote-gcrypt", - "liblockfile-simple-perl", "cabal-install", "vim", "less"] + "liblockfile-simple-perl", "cabal-install", "vim", "less", + "alex", "happy", "c2hs", + -- these haskell libs depend on C libs and don't use TH + "libghc-dbus-dev", "libghc-fdo-notify-dev", "libghc-network-protocol-xmpp-dev" + ] -- Installs current versions of git-annex's deps from cabal, but only -- does so once. -- cgit v1.3-2-g0d8e From 7ac991635c0dbdb2ace19480fcac22dae36e5409 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 21 May 2014 12:58:53 -0400 Subject: propellor spin --- config-joey.hs | 10 +++---- .../Property/SiteSpecific/GitAnnexBuilder.hs | 32 ++++++++-------------- 2 files changed, 16 insertions(+), 26 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index f495004e..386e5900 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - & Docker.docked hosts "armel-git-annex-builder-companion" - & Docker.docked hosts "armel-git-annex-builder" + ! Docker.docked hosts "armel-git-annex-builder-companion" + ! Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -199,7 +199,7 @@ hosts = -- (o) ` & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades -- This volume is shared with the armel builder. - & Docker.volume GitAnnexBuilder.homedir + & Docker.volume GitAnnexBuilder.gitbuilderdir -- Install current versions of build deps from cabal. & GitAnnexBuilder.tree "armel" & GitAnnexBuilder.buildDepsNoHaskellLibs @@ -208,8 +208,7 @@ hosts = -- (o) ` -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, & Docker.expose "22" & Apt.serviceInstalledRunning "ssh" - -- ssh key is shared via the home directory volume - & GitAnnexBuilder.sshKeyGen + & Ssh.authorizedKeys GitAnnexBuilder.builduser , Docker.container "armel-git-annex-builder" (image $ System (Debian Unstable) "armel") & Apt.stdSourcesList Unstable @@ -217,6 +216,7 @@ hosts = -- (o) ` & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" & Docker.volumes_from "armel-git-annex-builder-companion" + & Ssh.keyImported SshRsa GitAnnexBuilder.builduser -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 428b3d00..acf122af 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -53,15 +53,18 @@ builder' buildepsprop buildarch crontimes timeout rsyncupload = combinePropertie tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" [ User.accountFor builduser - -- Home directory already exists when docker volume is used. - , File.ownerGroup homedir builduser builduser , Apt.installed ["git"] - , check (not <$> doesDirectoryExist gitbuilderdir) $ userScriptProperty builduser - [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir - , "cd " ++ gitbuilderdir - , "git checkout " ++ buildarch - ] - `describe` "gitbuilder setup" + -- gitbuilderdir directory already exists when docker volume is used, + -- but with wrong owner. + , File.dirExists gitbuilderdir + , File.ownerGroup gitbuilderdir builduser builduser + , check (not <$> (doesDirectoryExist (gitbuilderdir ".git"))) $ + userScriptProperty builduser + [ "git clone git://git.kitenet.net/gitannexbuilder " ++ gitbuilderdir + , "cd " ++ gitbuilderdir + , "git checkout " ++ buildarch + ] + `describe` "gitbuilder setup" , check (not <$> doesDirectoryExist builddir) $ userScriptProperty builduser [ "git clone git://git-annex.branchable.com/ " ++ builddir ] @@ -90,16 +93,3 @@ cabalDeps = flagFile go cabalupdated where go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" - --- Ensure a ssh key is set up, and allow it to be used to ssh in -sshKeyGen :: Property -sshKeyGen = combineProperties "sshkeygen" - [ flagFile gen privkey - , flagFile auth authkeys - ] - where - gen = userScriptProperty builduser ["ssh-keygen -t RSA -N '' -f " ++ privkey] - auth = userScriptProperty builduser ["cp " ++ pubkey ++ " " ++ authkeys] - privkey = homedir ".ssh" "id_rsa" - pubkey = privkey ++ ".pub" - authkeys = homedir ".ssh" "authorized_keys" -- cgit v1.3-2-g0d8e From 497eb871e15683597b86619146db2364ff0d2cc4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 21 May 2014 13:07:36 -0400 Subject: propellor spin --- config-joey.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 386e5900..68bbcbb5 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -94,8 +94,8 @@ hosts = -- (o) ` & Docker.configured & Docker.docked hosts "amd64-git-annex-builder" & Docker.docked hosts "i386-git-annex-builder" - ! Docker.docked hosts "armel-git-annex-builder-companion" - ! Docker.docked hosts "armel-git-annex-builder" + & Docker.docked hosts "armel-git-annex-builder-companion" + & Docker.docked hosts "armel-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily -- cgit v1.3-2-g0d8e From 58e85f40197b3812efa39f214cd8f9149a4e39c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 21 May 2014 14:00:53 -0400 Subject: propellor spin --- config-joey.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 68bbcbb5..318d5d94 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -216,12 +216,12 @@ hosts = -- (o) ` & Apt.installed ["openssh-client"] & Docker.link "armel-git-annex-builder-companion" "companion" & Docker.volumes_from "armel-git-annex-builder-companion" - & Ssh.keyImported SshRsa GitAnnexBuilder.builduser -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) & GitAnnexBuilder.builder' GitAnnexBuilder.buildDepsNoHaskellLibs "armel" "1 3 * * *" "5h" True + & Ssh.keyImported SshRsa GitAnnexBuilder.builduser ] ++ monsters standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host -- cgit v1.3-2-g0d8e From 424ba1229ba0a786d6090d69739defe47d0612c8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 22 May 2014 21:42:57 -0400 Subject: refactor --- config-joey.hs | 54 ++++------------------ .../Property/SiteSpecific/GitAnnexBuilder.hs | 44 ++++++++++++++++++ 2 files changed, 53 insertions(+), 45 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 318d5d94..fafc409c 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -189,48 +189,12 @@ hosts = -- (o) ` (Just "remotes/origin/old-kitenet.net") -- git-annex autobuilder containers - , standardGitAnnexBuilder "amd64" 15 "2h" - , standardGitAnnexBuilder "i386" 45 "2h" - -- armel builder has a companion container using amd64 that - -- runs the build first to get TH splices. They need - -- to have the same versions of all haskell libraries installed. - , Docker.container "armel-git-annex-builder-companion" - (image $ System (Debian Unstable) "amd64") - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - -- This volume is shared with the armel builder. - & Docker.volume GitAnnexBuilder.gitbuilderdir - -- Install current versions of build deps from cabal. - & GitAnnexBuilder.tree "armel" - & GitAnnexBuilder.buildDepsNoHaskellLibs - & GitAnnexBuilder.cabalDeps - -- The armel builder can ssh to this companion, - -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, - & Docker.expose "22" - & Apt.serviceInstalledRunning "ssh" - & Ssh.authorizedKeys GitAnnexBuilder.builduser - , Docker.container "armel-git-annex-builder" - (image $ System (Debian Unstable) "armel") - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - & Apt.installed ["openssh-client"] - & Docker.link "armel-git-annex-builder-companion" "companion" - & Docker.volumes_from "armel-git-annex-builder-companion" - -- TODO: automate installing haskell libs - -- (Currently have to run - -- git-annex/standalone/linux/install-haskell-packages - -- which is not fully automated.) - & GitAnnexBuilder.builder' GitAnnexBuilder.buildDepsNoHaskellLibs "armel" "1 3 * * *" "5h" True - & Ssh.keyImported SshRsa GitAnnexBuilder.builduser + , GitAnnexBuilder.standardContainer dockerImage "amd64" 15 "2h" + , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" + , GitAnnexBuilder.armelCompanionContainer dockerImage + , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" ] ++ monsters -standardGitAnnexBuilder :: Architecture -> Int -> GitAnnexBuilder.TimeOut -> Host -standardGitAnnexBuilder arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") - (image $ System (Debian Unstable) arch) - & Apt.stdSourcesList Unstable - & Apt.unattendedUpgrades - & GitAnnexBuilder.builder arch (show buildminute ++ " * * * *") timeout True - -- This is my standard system setup. standardSystem :: HostName -> DebianSuite -> Architecture -> Host standardSystem hn suite arch = host hn @@ -257,7 +221,7 @@ standardSystem hn suite arch = host hn -- This is my standard container setup, featuring automatic upgrades. standardContainer :: Docker.ContainerName -> DebianSuite -> Architecture -> Host -standardContainer name suite arch = Docker.container name (image system) +standardContainer name suite arch = Docker.container name (dockerImage system) & os (System (Debian suite) arch) & Apt.stdSourcesList suite & Apt.unattendedUpgrades @@ -265,10 +229,10 @@ standardContainer name suite arch = Docker.container name (image system) system = System (Debian suite) arch -- Docker images I prefer to use. -image :: System -> Docker.Image -image (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch -image (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch -image _ = "debian-stable-official" -- does not currently exist! +dockerImage :: System -> Docker.Image +dockerImage (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch +dockerImage (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch +dockerImage _ = "debian-stable-official" -- does not currently exist! -- Digital Ocean does not provide any way to boot -- the kernel provided by the distribution, except using kexec. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index acf122af..80877fa5 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -4,7 +4,9 @@ import Propellor import qualified Propellor.Property.Apt as Apt import qualified Propellor.Property.User as User import qualified Propellor.Property.Cron as Cron +import qualified Propellor.Property.Ssh as Ssh import qualified Propellor.Property.File as File +import qualified Propellor.Property.Docker as Docker import Propellor.Property.Cron (CronTimes) builduser :: UserName @@ -93,3 +95,45 @@ cabalDeps = flagFile go cabalupdated where go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" + +standardContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host +standardContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") + (dockerImage $ System (Debian Unstable) arch) + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + & builder arch (show buildminute ++ " * * * *") timeout True + +-- armel builder has a companion container using amd64 that +-- runs the build first to get TH splices. They need +-- to have the same versions of all haskell libraries installed. +armelCompanionContainer :: (System -> Docker.Image) -> Host +armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder-companion" + (dockerImage $ System (Debian Unstable) "amd64") + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + -- This volume is shared with the armel builder. + & Docker.volume gitbuilderdir + -- Install current versions of build deps from cabal. + & tree "armel" + & buildDepsNoHaskellLibs + & cabalDeps + -- The armel builder can ssh to this companion, + -- using $COMPANION_PORT_22_TCP_ADDR as the hostname, + & Docker.expose "22" + & Apt.serviceInstalledRunning "ssh" + & Ssh.authorizedKeys builduser + +armelContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" + (dockerImage $ System (Debian Unstable) "armel") + & Apt.stdSourcesList Unstable + & Apt.unattendedUpgrades + & Apt.installed ["openssh-client"] + & Docker.link "armel-git-annex-builder-companion" "companion" + & Docker.volumes_from "armel-git-annex-builder-companion" + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/linux/install-haskell-packages + -- which is not fully automated.) + & builder' buildDepsNoHaskellLibs "armel" crontimes timeout True + & Ssh.keyImported SshRsa builduser -- cgit v1.3-2-g0d8e From 0ec565a590e0b5e70af4f4f922925e5b0b718376 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 23 May 2014 12:30:25 -0400 Subject: propellor spin --- config-joey.hs | 37 +++------------------- debian/changelog | 1 + propellor.cabal | 2 ++ .../Property/HostingProvider/CloudAtCost.hs | 24 ++++++++++++++ .../Property/HostingProvider/DigitalOcean.hs | 21 ++++++++++++ .../Property/SiteSpecific/GitAnnexBuilder.hs | 2 +- 6 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 src/Propellor/Property/HostingProvider/CloudAtCost.hs create mode 100644 src/Propellor/Property/HostingProvider/DigitalOcean.hs (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index fafc409c..2f84fbb0 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -21,6 +21,8 @@ import qualified Propellor.Property.Git as Git import qualified Propellor.Property.Apache as Apache import qualified Propellor.Property.Postfix as Postfix import qualified Propellor.Property.Service as Service +import qualified Propellor.Property.HostingProvider.DigitalOcean as DigitalOcean +import qualified Propellor.Property.HostingProvider.CloudAtCost as CloudAtCost import qualified Propellor.Property.SiteSpecific.GitHome as GitHome import qualified Propellor.Property.SiteSpecific.GitAnnexBuilder as GitAnnexBuilder import qualified Propellor.Property.SiteSpecific.JoeySites as JoeySites @@ -46,7 +48,7 @@ hosts = -- (o) ` & ipv4 "162.248.143.249" & ipv6 "2002:5044:5531::1" - & cleanCloudAtCost + & CloudAtCost.decruft & Apt.unattendedUpgrades & Network.ipv6to4 & Tor.isBridge @@ -103,6 +105,7 @@ hosts = -- (o) ` , standardSystem "diatom.kitenet.net" Stable "amd64" & ipv4 "107.170.31.195" + & DigitalOcean.distroKernel & Hostname.sane & Ssh.hostKey SshDsa & Ssh.hostKey SshRsa @@ -234,38 +237,6 @@ dockerImage (System (Debian Unstable) arch) = "joeyh/debian-unstable-" ++ arch dockerImage (System (Debian Stable) arch) = "joeyh/debian-stable-" ++ arch dockerImage _ = "debian-stable-official" -- does not currently exist! --- Digital Ocean does not provide any way to boot --- the kernel provided by the distribution, except using kexec. --- Without this, some old, and perhaps insecure kernel will be used. --- --- Note that this only causes the new kernel to be loaded on reboot. --- If the power is cycled, the old kernel still boots up. --- TODO: detect this and reboot immediately? -digitalOceanDistroKernel :: Property -digitalOceanDistroKernel = propertyList "digital ocean distro kernel hack" - [ Apt.installed ["grub-pc", "kexec-tools"] - , "/etc/default/kexec" `File.containsLines` - [ "LOAD_KEXEC=true" - , "USE_GRUB_CONFIG=true" - ] - ] - --- Clean up a system as installed by cloudatcost.com -cleanCloudAtCost :: Property -cleanCloudAtCost = propertyList "cloudatcost cleanup" - [ Hostname.sane - , Ssh.randomHostKeys - , "worked around grub/lvm boot bug #743126" ==> - "/etc/default/grub" `File.containsLine` "GRUB_DISABLE_LINUX_UUID=true" - `onChange` cmdProperty "update-grub" [] - `onChange` cmdProperty "update-initramfs" ["-u"] - , combineProperties "nuked cloudatcost cruft" - [ File.notPresent "/etc/rc.local" - , File.notPresent "/etc/init.d/S97-setup.sh" - , User.nuked "user" User.YesReallyDeleteHome - ] - ] - myDnsSecondary :: Property myDnsSecondary = propertyList "dns secondary for all my domains" $ map toProp [ Dns.secondary hosts "kitenet.net" diff --git a/debian/changelog b/debian/changelog index 9d1d9a1c..a3dc1032 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ propellor (0.5.3) UNRELEASED; urgency=medium * Fix unattended-upgrades config for !stable. * Ensure that kernel hostname is same as /etc/hostname when configuring hostname. + * Added modules for some hosting providers (DigitalOcean, CloudAtCost). -- Joey Hess Sun, 18 May 2014 13:44:00 -0400 diff --git a/propellor.cabal b/propellor.cabal index 9d0612e9..da9a630f 100644 --- a/propellor.cabal +++ b/propellor.cabal @@ -92,6 +92,8 @@ Library Propellor.Property.Sudo Propellor.Property.Tor Propellor.Property.User + Propellor.Property.HostingProvider.DigitalOcean + Propellor.Property.HostingProvider.CloudAtCost Propellor.Property.SiteSpecific.GitHome Propellor.Property.SiteSpecific.JoeySites Propellor.Property.SiteSpecific.GitAnnexBuilder diff --git a/src/Propellor/Property/HostingProvider/CloudAtCost.hs b/src/Propellor/Property/HostingProvider/CloudAtCost.hs new file mode 100644 index 00000000..003bd3c5 --- /dev/null +++ b/src/Propellor/Property/HostingProvider/CloudAtCost.hs @@ -0,0 +1,24 @@ +module Propellor.Property.HostingProvider.CloudAtCost where + +import Propellor +import qualified Propellor.Property.Hostname as Hostname +import qualified Propellor.Property.File as File +import qualified Propellor.Property.Ssh as Ssh +import qualified Propellor.Property.User as User + +-- Clean up a system as installed by cloudatcost.com +decruft :: Property +decruft = propertyList "cloudatcost cleanup" + [ Hostname.sane + , Ssh.randomHostKeys + , "worked around grub/lvm boot bug #743126" ==> + "/etc/default/grub" `File.containsLine` "GRUB_DISABLE_LINUX_UUID=true" + `onChange` cmdProperty "update-grub" [] + `onChange` cmdProperty "update-initramfs" ["-u"] + , combineProperties "nuked cloudatcost cruft" + [ File.notPresent "/etc/rc.local" + , File.notPresent "/etc/init.d/S97-setup.sh" + , User.nuked "user" User.YesReallyDeleteHome + ] + ] + diff --git a/src/Propellor/Property/HostingProvider/DigitalOcean.hs b/src/Propellor/Property/HostingProvider/DigitalOcean.hs new file mode 100644 index 00000000..24dfd356 --- /dev/null +++ b/src/Propellor/Property/HostingProvider/DigitalOcean.hs @@ -0,0 +1,21 @@ +module Propellor.Property.HostingProvider.DigitalOcean where + +import Propellor +import qualified Propellor.Property.Apt as Apt +import qualified Propellor.Property.File as File + +-- Digital Ocean does not provide any way to boot +-- the kernel provided by the distribution, except using kexec. +-- Without this, some old, and perhaps insecure kernel will be used. +-- +-- Note that this only causes the new kernel to be loaded on reboot. +-- If the power is cycled, the old kernel still boots up. +-- TODO: detect this and reboot immediately? +distroKernel :: Property +distroKernel = propertyList "digital ocean distro kernel hack" + [ Apt.installed ["grub-pc", "kexec-tools"] + , "/etc/default/kexec" `File.containsLines` + [ "LOAD_KEXEC=true" + , "USE_GRUB_CONFIG=true" + ] + ] diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 56123b60..8055a608 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -140,4 +140,4 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex where writecompanionaddress = scriptProperty [ "echo \"$COMPANION_PORT_22_TCP_ADDR\" > " ++ homedir "companion_address" - ] + ] `describe` "companion_address file" -- cgit v1.3-2-g0d8e From 60ea130321e23dd7b36337388d6627661c116548 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 24 May 2014 00:05:47 -0400 Subject: propellor spin --- config-joey.hs | 2 ++ src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 2f84fbb0..1f5ce0dc 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -98,6 +98,7 @@ hosts = -- (o) ` & Docker.docked hosts "i386-git-annex-builder" & Docker.docked hosts "armel-git-annex-builder-companion" & Docker.docked hosts "armel-git-annex-builder" + & Docker.docked hosts "android-git-annex-builder" & Docker.garbageCollected `period` Daily & Apt.buildDep ["git-annex"] `period` Daily @@ -196,6 +197,7 @@ hosts = -- (o) ` , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" , GitAnnexBuilder.armelCompanionContainer dockerImage , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" + , GitAnnexBuilder.androidContainer dockerImage "1 1 * * *" "3h" ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 8055a608..13f1ac12 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -103,6 +103,24 @@ standardContainer dockerImage arch buildminute timeout = Docker.container (arch & Apt.unattendedUpgrades & builder arch (show buildminute ++ " * * * *") timeout True +-- Android is cross-built in a Debian i386 container, using the Android NDK. +androidContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +androidContainer dockerImage crontimes timeout = Docker.container "android-git-annex-builder" + (dockerImage $ System (Debian Stable) "i386") + & Apt.stdSourcesList Stable + & Apt.unattendedUpgrades + & builder "android" crontimes timeout True + -- Use git-annex's android chroot setup script, which will install + -- ghc-android and the NDK, etc, in the home directory of the + -- builder user. + & scriptProperty + [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" + ] + -- TODO: automate installing haskell libs + -- (Currently have to run + -- git-annex/standalone/android/install-haskell-packages + -- which is not fully automated.) + -- armel builder has a companion container using amd64 that -- runs the build first to get TH splices. They need -- to have the same versions of all haskell libraries installed. -- cgit v1.3-2-g0d8e From c9a20304ccbbe7ffaba1d6b3bfc45599bfc06d14 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 24 May 2014 16:51:46 -0400 Subject: propellor spin --- config-joey.hs | 1 - 1 file changed, 1 deletion(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 1f5ce0dc..84cc7167 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -302,7 +302,6 @@ monsters = -- but do want to track their public keys etc. & alias "wortroot.kitenet.net" & alias "www.wortroot.kitenet.net" & alias "joey.kitenet.net" - & alias "annex.kitenet.net" & alias "ipv6.kitenet.net" & alias "bitlbee.kitenet.net" , host "mouse.kitenet.net" -- cgit v1.3-2-g0d8e From 16fbf3c6f67f1eed11a9d189e2c7c2723ef5969a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 27 May 2014 21:17:53 -0400 Subject: reorg git-annex android builder, and put a dev container on my laptop --- config-joey.hs | 13 +++++--- .../Property/SiteSpecific/GitAnnexBuilder.hs | 35 ++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 84cc7167..6c4dff7b 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -42,6 +42,7 @@ hosts = -- (o) ` & ipv6 "2001:4830:1600:187::2" -- sixxs tunnel & Docker.configured & Apt.buildDep ["git-annex"] `period` Daily + & Docker.docked hosts "android-git-annex" -- Nothing super-important lives here. , standardSystem "clam.kitenet.net" Unstable "amd64" @@ -193,11 +194,15 @@ hosts = -- (o) ` (Just "remotes/origin/old-kitenet.net") -- git-annex autobuilder containers - , GitAnnexBuilder.standardContainer dockerImage "amd64" 15 "2h" - , GitAnnexBuilder.standardContainer dockerImage "i386" 45 "2h" + , GitAnnexBuilder.standardAutoBuilderContainer dockerImage "amd64" 15 "2h" + , GitAnnexBuilder.standardAutoBuilderContainer dockerImage "i386" 45 "2h" , GitAnnexBuilder.armelCompanionContainer dockerImage - , GitAnnexBuilder.armelContainer dockerImage "1 3 * * *" "5h" - , GitAnnexBuilder.androidContainer dockerImage "1 1 * * *" "3h" + , GitAnnexBuilder.armelAutoBuilderContainer dockerImage "1 3 * * *" "5h" + , GitAnnexBuilder.androidAutoBuilderContainer dockerImage "1 1 * * *" "3h" + + -- for development of git-annex for android + , GitAnnexBuilder.androidContainer dockerImage "android-git-annex" + & Docker.volume ("/home/joey/src/git-annex:" ++ GitAnnexBuilder.homedir "git-annex") ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index a154c926..15e4e6ce 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -24,12 +24,8 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder = builder' buildDeps - -builder' :: Property -> Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder' buildepsprop buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" +builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" [ tree buildarch - , buildepsprop , Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" @@ -72,8 +68,8 @@ tree buildarch = combineProperties "gitannexbuilder tree" ] ] -buildDeps :: Property -buildDeps = combineProperties "gitannexbuilder build deps" +buildDepsApt :: Property +buildDepsApt = combineProperties "gitannexbuilder build deps" [ Apt.buildDep ["git-annex"] , buildDepsFewHaskellLibs , "git-annex source build deps installed" ==> Apt.buildDepIn builddir @@ -102,20 +98,26 @@ cabalDeps = flagFile go cabalupdated go = userScriptProperty builduser ["cabal update && cabal install git-annex --only-dependencies || true"] cabalupdated = homedir ".cabal" "packages" "hackage.haskell.org" "00-index.cache" -standardContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host -standardContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") +standardAutoBuilderContainer :: (System -> Docker.Image) -> Architecture -> Int -> TimeOut -> Host +standardAutoBuilderContainer dockerImage arch buildminute timeout = Docker.container (arch ++ "-git-annex-builder") (dockerImage $ System (Debian Unstable) arch) & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades + & buildDepsApt & builder arch (show buildminute ++ " * * * *") timeout True +androidAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +androidAutoBuilderContainer dockerImage crontimes timeout = + androidContainer dockerImage "android-git-annex-builder" + & Apt.unattendedUpgrades + & builder "android" crontimes timeout True + -- Android is cross-built in a Debian i386 container, using the Android NDK. -androidContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host -androidContainer dockerImage crontimes timeout = Docker.container "android-git-annex-builder" +androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Host +androidContainer dockerImage name = Docker.container name (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable - & Apt.unattendedUpgrades - & builder' buildDepsNoHaskellLibs "android" crontimes timeout True + & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") -- TODO: automate installing haskell libs -- (Currently have to run @@ -148,8 +150,8 @@ armelCompanionContainer dockerImage = Docker.container "armel-git-annex-builder- & Apt.serviceInstalledRunning "ssh" & Ssh.authorizedKeys builduser -armelContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host -armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" +armelAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host +armelAutoBuilderContainer dockerImage crontimes timeout = Docker.container "armel-git-annex-builder" (dockerImage $ System (Debian Unstable) "armel") & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades @@ -160,7 +162,8 @@ armelContainer dockerImage crontimes timeout = Docker.container "armel-git-annex -- (Currently have to run -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) - & builder' buildDepsFewHaskellLibs "armel" crontimes timeout True + & buildDepsFewHaskellLibs + & builder "armel" crontimes timeout True & Ssh.keyImported SshRsa builduser & trivial writecompanionaddress where -- cgit v1.3-2-g0d8e From a0b808e8923c2ceb598bf56f8f2628c802574765 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 28 May 2014 01:32:54 -0400 Subject: comment fun --- config-joey.hs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index 6c4dff7b..e145a39b 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -27,16 +27,13 @@ import qualified Propellor.Property.SiteSpecific.GitHome as GitHome import qualified Propellor.Property.SiteSpecific.GitAnnexBuilder as GitAnnexBuilder import qualified Propellor.Property.SiteSpecific.JoeySites as JoeySites -main :: IO () -main = defaultMain hosts - - -- _ ______`| ,-.__ - {- Propellor -- / \___-=O`/|O`/__| (____.' - Deployed -} -- \ / | / ) _.-"-._ - -- `/-==__ _/__|/__=-| ( \_ -hosts :: [Host] -- * \ | | '--------' -hosts = -- (o) ` +main :: IO () -- _ ______`| ,-.__ +main = defaultMain hosts -- / \___-=O`/|O`/__| (____.' + {- Propellor -- \ / | / ) _.-"-._ + Deployed -} -- `/-==__ _/__|/__=-| ( \_ +hosts :: [Host] -- * \ | | '--------' +hosts = -- (o) ` -- My laptop [ host "darkstar.kitenet.net" & ipv6 "2001:4830:1600:187::2" -- sixxs tunnel -- cgit v1.3-2-g0d8e From 2066292be9490fe2d8bc433e4c21b0a1907455e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 28 May 2014 10:38:29 -0400 Subject: propellor spin --- config-joey.hs | 8 ++++--- src/Propellor/Property.hs | 3 +++ .../Property/SiteSpecific/GitAnnexBuilder.hs | 28 ++++++++++++---------- 3 files changed, 23 insertions(+), 16 deletions(-) (limited to 'config-joey.hs') diff --git a/config-joey.hs b/config-joey.hs index e145a39b..7d0a4e80 100644 --- a/config-joey.hs +++ b/config-joey.hs @@ -197,9 +197,11 @@ hosts = -- (o) ` , GitAnnexBuilder.armelAutoBuilderContainer dockerImage "1 3 * * *" "5h" , GitAnnexBuilder.androidAutoBuilderContainer dockerImage "1 1 * * *" "3h" - -- for development of git-annex for android - , GitAnnexBuilder.androidContainer dockerImage "android-git-annex" - & Docker.volume ("/home/joey/src/git-annex:" ++ GitAnnexBuilder.homedir "git-annex") + -- for development of git-annex for android, using my git-annex + -- work tree + , let gitannexdir = GitAnnexBuilder.homedir "git-annex" + in GitAnnexBuilder.androidContainer dockerImage "android-git-annex" doNothing gitannexdir + & Docker.volume ("/home/joey/src/git-annex:" ++ gitannexdir) ] ++ monsters -- This is my standard system setup. diff --git a/src/Propellor/Property.hs b/src/Propellor/Property.hs index 24494654..0728932e 100644 --- a/src/Propellor/Property.hs +++ b/src/Propellor/Property.hs @@ -103,6 +103,9 @@ trivial p = adjustProperty p $ \satisfy -> do then return NoChange else return r +doNothing :: Property +doNothing = property "noop property" noChange + -- | Makes a property that is satisfied differently depending on the host's -- operating system. -- diff --git a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs index 15e4e6ce..4f086a70 100644 --- a/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs +++ b/src/Propellor/Property/SiteSpecific/GitAnnexBuilder.hs @@ -23,10 +23,9 @@ builddir = gitbuilderdir "build" type TimeOut = String -- eg, 5h -builder :: Architecture -> CronTimes -> TimeOut -> Bool -> Property -builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbuilder" - [ tree buildarch - , Apt.serviceInstalledRunning "cron" +autobuilder :: CronTimes -> TimeOut -> Bool -> Property +autobuilder crontimes timeout rsyncupload = combineProperties "gitannexbuilder" + [ Apt.serviceInstalledRunning "cron" , Cron.niceJob "gitannexbuilder" crontimes builduser gitbuilderdir $ "git pull ; timeout " ++ timeout ++ " ./autobuild" -- The builduser account does not have a password set, @@ -50,8 +49,7 @@ builder buildarch crontimes timeout rsyncupload = combineProperties "gitannexbui tree :: Architecture -> Property tree buildarch = combineProperties "gitannexbuilder tree" - [ User.accountFor builduser - , Apt.installed ["git"] + [ Apt.installed ["git"] -- gitbuilderdir directory already exists when docker volume is used, -- but with wrong owner. , File.dirExists gitbuilderdir @@ -104,21 +102,24 @@ standardAutoBuilderContainer dockerImage arch buildminute timeout = Docker.conta & Apt.stdSourcesList Unstable & Apt.unattendedUpgrades & buildDepsApt - & builder arch (show buildminute ++ " * * * *") timeout True + & autobuilder (show buildminute ++ " * * * *") timeout True + `requires` tree arch androidAutoBuilderContainer :: (System -> Docker.Image) -> Cron.CronTimes -> TimeOut -> Host androidAutoBuilderContainer dockerImage crontimes timeout = - androidContainer dockerImage "android-git-annex-builder" + androidContainer dockerImage "android-git-annex-builder" (tree "android") builddir & Apt.unattendedUpgrades - & builder "android" crontimes timeout True + & autobuilder crontimes timeout True -- Android is cross-built in a Debian i386 container, using the Android NDK. -androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Host -androidContainer dockerImage name = Docker.container name +androidContainer :: (System -> Docker.Image) -> Docker.ContainerName -> Property -> FilePath -> Host +androidContainer dockerImage name setupgitannexdir gitannexdir = Docker.container name (dockerImage $ System (Debian Stable) "i386") & Apt.stdSourcesList Stable + & User.accountFor builduser & buildDepsNoHaskellLibs & flagFile chrootsetup ("/chrootsetup") + `requires` setupgitannexdir -- TODO: automate installing haskell libs -- (Currently have to run -- git-annex/standalone/android/install-haskell-packages @@ -128,7 +129,7 @@ androidContainer dockerImage name = Docker.container name -- ghc-android and the NDK, all build deps, etc, in the home -- directory of the builder user. chrootsetup = scriptProperty - [ "cd " ++ builddir ++ " && ./standalone/android/buildchroot-inchroot" + [ "cd " ++ gitannexdir ++ " && ./standalone/android/buildchroot-inchroot" ] -- armel builder has a companion container using amd64 that @@ -163,7 +164,8 @@ armelAutoBuilderContainer dockerImage crontimes timeout = Docker.container "arme -- git-annex/standalone/linux/install-haskell-packages -- which is not fully automated.) & buildDepsFewHaskellLibs - & builder "armel" crontimes timeout True + & autobuilder crontimes timeout True + `requires` tree "armel" & Ssh.keyImported SshRsa builduser & trivial writecompanionaddress where -- cgit v1.3-2-g0d8e