Something went wrong. Try again.
forked from tailscale/tailscale
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168// Copyright (c) Tailscale Inc & contributors// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import ( "bytes" "context" _ "embed" "encoding/base64" "encoding/json" "encoding/pem" "errors" "fmt" "io" "io/fs" "maps" "net" "net/http" "net/http/httptest" "net/netip" "os" "os/exec" "path/filepath" "strconv" "strings" "sync" "syscall" "testing" "time"
"github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" "tailscale.com/client/local" "tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/health" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" "tailscale.com/net/memnet" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/types/key" "tailscale.com/types/views")
const configFileAuthKey = "some-auth-key"
func TestContainerBoot(t *testing.T) { flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/19380") boot := filepath.Join(t.TempDir(), "containerboot") if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil { t.Fatalf("Building containerboot: %v", err) } egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.2") egressStatusUpdated := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.3")
metricsURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d/metrics", port) } healthURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d/healthz", port) } egressSvcTerminateURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP) }
capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
type phase struct { // If non-nil, send this IPN bus notification (and remember it as the // initial update for any future new watchers, then wait for all the // Waits below to be true before proceeding to the next phase. Notify *ipn.Notify
// WantCmds is the commands that containerboot should run in this phase. WantCmds []string
// WantKubeSecret is the secret keys/values that should exist in the // kube secret. WantKubeSecret map[string]string
// Update the kube secret with these keys/values at the beginning of the // phase (simulates our fake tailscaled doing it). UpdateKubeSecret map[string]string
// Update files with these paths/contents at the beginning of the phase // (simulates the operator updating mounted config files). UpdateFiles map[string]string
// WantFiles files that should exist in the container and their // contents. WantFiles map[string]string
// WantLog is a log message we expect from containerboot. WantLog string
// If set for a phase, the test will expect containerboot to exit with // this error code, and the test will finish on that phase without // waiting for the successful startup log message. WantExitCode *int
// The signal to send to containerboot at the start of the phase. Signal *syscall.Signal
EndpointStatuses map[string]int } runningNotify := &ipn.Notify{ State: new(ipn.Running), SelfChange: &tailcfg.Node{ StableID: tailcfg.StableNodeID("myID"), Name: "test-node.test.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, } type testCase struct { Env map[string]string KubeSecret map[string]string KubeDenyPatch bool Phases []phase } tests := map[string]func(env *testEnv) testCase{ "no_args": func(env *testEnv) testCase { return testCase{ // Out of the box default: runs in userspace mode, ephemeral storage, interactive login. Env: nil, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, // No metrics or health by default. EndpointStatuses: map[string]int{ metricsURL(9002): -1, healthURL(9002): -1, }, }, { Notify: runningNotify, }, }, } }, "authkey": func(env *testEnv) testCase { return testCase{ // Userspace mode, ephemeral storage, authkey provided on every run. Env: map[string]string{ "TS_AUTHKEY": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "authkey_old_flag": func(env *testEnv) testCase { return testCase{ // Userspace mode, ephemeral storage, authkey provided on every run. Env: map[string]string{ "TS_AUTH_KEY": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "authkey_disk_state": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_STATE_DIR": filepath.Join(env.d, "tmp"), }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "routes": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_ROUTES": "1.2.3.0/24,10.20.30.0/24", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24", }, }, { Notify: runningNotify, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "0", "proc/sys/net/ipv6/conf/all/forwarding": "0", }, }, }, } }, "empty_routes": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_ROUTES": "", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=", }, }, { Notify: runningNotify, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "0", "proc/sys/net/ipv6/conf/all/forwarding": "0", }, }, }, } }, "routes_kernel_ipv4": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_ROUTES": "1.2.3.0/24,10.20.30.0/24", "TS_USERSPACE": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24", }, }, { Notify: runningNotify, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "1", "proc/sys/net/ipv6/conf/all/forwarding": "0", }, }, }, } }, "routes_kernel_ipv6": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_ROUTES": "::/64,1::/64", "TS_USERSPACE": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64", }, }, { Notify: runningNotify, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "0", "proc/sys/net/ipv6/conf/all/forwarding": "1", }, }, }, } }, "routes_kernel_all_families": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_ROUTES": "::/64,1.2.3.0/24", "TS_USERSPACE": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24", }, }, { Notify: runningNotify, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "1", "proc/sys/net/ipv6/conf/all/forwarding": "1", }, }, }, } }, "ingress_proxy": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_DEST_IP": "1.2.3.4", "TS_USERSPACE": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "egress_proxy": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_TAILNET_TARGET_IP": "100.99.99.99", "TS_USERSPACE": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "1", "proc/sys/net/ipv6/conf/all/forwarding": "0", }, }, { Notify: runningNotify, }, }, } }, "egress_proxy_fqdn_ipv6_target_on_ipv4_host": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_TAILNET_TARGET_FQDN": "ipv6-node.test.ts.net.", // resolves to IPv6 address "TS_USERSPACE": "false", "TS_TEST_FAKE_NETFILTER_6": "false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantFiles: map[string]string{ "proc/sys/net/ipv4/ip_forward": "1", "proc/sys/net/ipv6/conf/all/forwarding": "0", }, }, { Notify: &ipn.Notify{ State: new(ipn.Running), SelfChange: &tailcfg.Node{ StableID: tailcfg.StableNodeID("myID"), Name: "test-node.test.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, PeersChanged: []*tailcfg.Node{ { StableID: tailcfg.StableNodeID("ipv6ID"), Name: "ipv6-node.test.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")}, }, }, }, WantLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false", WantExitCode: new(1), }, }, } }, "authkey_once": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_AUTH_ONCE": "true", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", }, }, { Notify: &ipn.Notify{ State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", }, }, }, } }, "auth_key_once_extra_args_override_dns": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", "TS_AUTH_ONCE": "true", "TS_ACCEPT_DNS": "false", "TS_EXTRA_ARGS": "--accept-dns", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", }, }, { Notify: &ipn.Notify{ State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true --authkey=tskey-key", }, }, { Notify: runningNotify, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=true", }, }, }, } }, "kube_storage": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "POD_UID": "some-pod-uid", }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, kubetypes.KeyPodUID: "some-pod-uid", }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, kubetypes.KeyPodUID: "some-pod-uid", }, }, }, } }, "kube_disk_storage": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, // Explicitly set to an empty value, to override the default of "tailscale". "TS_KUBE_SECRET": "", "TS_STATE_DIR": filepath.Join(env.d, "tmp"), "TS_AUTHKEY": "tskey-key", }, KubeSecret: map[string]string{}, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{}, }, { Notify: runningNotify, WantKubeSecret: map[string]string{}, }, }, } }, "kube_storage_no_patch": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "TS_AUTHKEY": "tskey-key", }, KubeSecret: map[string]string{}, KubeDenyPatch: true, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{}, }, { Notify: runningNotify, WantKubeSecret: map[string]string{}, }, }, } }, "kube_storage_auth_once": func(env *testEnv) testCase { return testCase{ // Same as previous, but deletes the authkey from the kube secret. Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "TS_AUTH_ONCE": "true", }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", }, WantKubeSecret: map[string]string{ "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, }, }, }, } }, "kube_storage_updates": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ State: new(ipn.Running), SelfChange: &tailcfg.Node{ StableID: tailcfg.StableNodeID("newID"), Name: "new-name.test.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "device_fqdn": "new-name.test.ts.net.", "device_id": "newID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, }, }, }, } }, "proxies": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_SOCKS5_SERVER": "localhost:1080", "TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, }, { Notify: runningNotify, }, }, } }, "dns": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_ACCEPT_DNS": "true", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true", }, }, { Notify: runningNotify, }, }, } }, "extra_args": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EXTRA_ARGS": "--widget=rotated", "TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated", }, }, { Notify: runningNotify, }, }, } }, "extra_args_accept_routes": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EXTRA_ARGS": "--accept-routes", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --accept-routes", }, }, { Notify: runningNotify, }, }, } }, "extra_args_accept_dns": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EXTRA_ARGS": "--accept-dns", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true", }, }, { Notify: runningNotify, }, }, } }, "extra_args_accept_dns_overrides_env_var": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_ACCEPT_DNS": "true", // Overridden by TS_EXTRA_ARGS. "TS_EXTRA_ARGS": "--accept-dns=false", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, }, { Notify: runningNotify, }, }, } }, "hostname": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_HOSTNAME": "my-server", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --hostname=my-server", }, }, { Notify: runningNotify, }, }, } }, "experimental_tailscaled_config_path": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", }, }, { Notify: runningNotify, }, }, } }, "sets_reissue_authkey_if_needs_login": func(env *testEnv) testCase { newAuthKey := "new-reissued-auth-key" return testCase{ Env: map[string]string{ "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, }, Phases: []phase{ { UpdateFiles: map[string]string{ "etc/tailscaled/..data": "", }, WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ State: new(ipn.NeedsLogin), }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, kubetypes.KeyReissueAuthkey: configFileAuthKey, }, WantLog: "watching for config changes via fsnotify", }, { UpdateFiles: map[string]string{ "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), "etc/tailscaled/..data": "updated", }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, }, WantExitCode: new(0), WantLog: "Successfully received new auth key, restarting to apply configuration", }, }, } }, "sets_reissue_authkey_if_auth_fails": func(env *testEnv) testCase { newAuthKey := "new-reissued-auth-key" return testCase{ Env: map[string]string{ "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, }, Phases: []phase{ { UpdateFiles: map[string]string{ "etc/tailscaled/..data": "", }, WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ Health: &health.State{ Warnings: map[health.WarnableCode]health.UnhealthyState{ health.LoginStateWarnable.Code: {}, }, }, }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, kubetypes.KeyReissueAuthkey: configFileAuthKey, }, WantLog: "watching for config changes via fsnotify", }, { UpdateFiles: map[string]string{ "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), "etc/tailscaled/..data": "updated", }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, }, WantExitCode: new(0), WantLog: "Successfully received new auth key, restarting to apply configuration", }, }, } }, "clears_reissue_authkey_on_change": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, }, KubeSecret: map[string]string{ kubetypes.KeyReissueAuthkey: "some-older-authkey", "foo": "bar", // Check not everything is cleared. }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", }, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, "foo": "bar", }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ kubetypes.KeyCapVer: capver, "foo": "bar", kubetypes.KeyDeviceFQDN: "test-node.test.ts.net.", kubetypes.KeyDeviceID: "myID", kubetypes.KeyDeviceIPs: `["100.64.0.1"]`, }, }, }, } }, "metrics_enabled": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort), "TS_ENABLE_METRICS": "true", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): 200, healthURL(env.localAddrPort): -1, }, }, { Notify: runningNotify, }, }, } }, "health_enabled": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort), "TS_ENABLE_HEALTH_CHECK": "true", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): -1, healthURL(env.localAddrPort): 503, // Doesn't start passing until the next phase. }, }, { Notify: runningNotify, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): -1, healthURL(env.localAddrPort): 200, }, }, }, } }, "metrics_and_health_on_same_port": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort), "TS_ENABLE_METRICS": "true", "TS_ENABLE_HEALTH_CHECK": "true", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): 200, healthURL(env.localAddrPort): 503, // Doesn't start passing until the next phase. }, }, { Notify: runningNotify, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): 200, healthURL(env.localAddrPort): 200, }, }, }, } }, "local_metrics_and_deprecated_health": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort), "TS_ENABLE_METRICS": "true", "TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", env.healthAddrPort), }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", }, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): 200, healthURL(env.healthAddrPort): 503, // Doesn't start passing until the next phase. }, }, { Notify: runningNotify, EndpointStatuses: map[string]int{ metricsURL(env.localAddrPort): 200, healthURL(env.healthAddrPort): 200, }, }, }, } }, "serve_config_no_kube": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config.json"), "TS_AUTHKEY": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "serve_config_kube": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config.json"), }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, "https_endpoint": "no-https", kubetypes.KeyCapVer: capver, }, }, }, } }, "egress_svcs_config_kube": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(env.d, "etc/tailscaled"), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort), }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, }, }, { Notify: &ipn.Notify{ State: new(ipn.Running), SelfChange: &tailcfg.Node{ StableID: tailcfg.StableNodeID("myID"), Name: "test-node.test.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, PeersChanged: []*tailcfg.Node{ { StableID: tailcfg.StableNodeID("fooID"), Name: "foo.tailnetxyz.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}, }, }, }, WantKubeSecret: map[string]string{ "egress-services": string(mustJSON(t, egressStatus)), "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, }, }, { Notify: &ipn.Notify{ PeersChanged: []*tailcfg.Node{{ StableID: tailcfg.StableNodeID("fooID"), Name: "foo.tailnetxyz.ts.net.", Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.3/32")}, }}, }, WantKubeSecret: map[string]string{ "egress-services": string(mustJSON(t, egressStatusUpdated)), "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net.", "device_id": "myID", "device_ips": `["100.64.0.1"]`, kubetypes.KeyCapVer: capver, }, }, }, } }, "egress_svcs_config_no_kube": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(env.d, "etc/tailscaled"), "TS_AUTHKEY": "tskey-key", }, Phases: []phase{ { WantLog: "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", WantExitCode: new(1), }, }, } }, "serve_config_with_service_auto_advertisement": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config-with-services.json"), "TS_AUTHKEY": "tskey-key", }, Phases: []phase{ { WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, }, { Notify: runningNotify, }, }, } }, "kube_shutdown_during_state_write": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "TS_ENABLE_HEALTH_CHECK": "true", }, KubeSecret: map[string]string{ "authkey": "tskey-key", }, Phases: []phase{ { // Normal startup. WantCmds: []string{ "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", kubetypes.KeyCapVer: capver, }, }, { // SIGTERM before state is finished writing, should wait for // consistent state before propagating SIGTERM to tailscaled. Signal: new(unix.SIGTERM), UpdateKubeSecret: map[string]string{ "_machinekey": "foo", "_profiles": "foo", "profile-baff": "foo", // Missing "_current-profile" key. }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "_machinekey": "foo", "_profiles": "foo", "profile-baff": "foo", kubetypes.KeyCapVer: capver, }, WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", }, { // tailscaled has finished writing state, should propagate SIGTERM. UpdateKubeSecret: map[string]string{ "_current-profile": "foo", }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "_machinekey": "foo", "_profiles": "foo", "profile-baff": "foo", "_current-profile": "foo", kubetypes.KeyCapVer: capver, }, WantLog: "HTTP server at [::]:9002 closed", WantExitCode: new(0), }, }, } }, }
for name, test := range tests { t.Run(name, func(t *testing.T) { t.Parallel() env := newTestEnv(t) tc := test(&env)
for k, v := range tc.KubeSecret { env.kube.SetSecret(k, v) } env.kube.SetPatching(!tc.KubeDenyPatch)
cmd := exec.Command(boot) cmd.Env = []string{ fmt.Sprintf("PATH=%s/usr/bin:%s", env.d, os.Getenv("PATH")), fmt.Sprintf("TS_TEST_RECORD_ARGS=%s", env.argFile), fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d), "TS_TEST_FAKE_NETFILTER=true", } for k, v := range tc.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } cbOut := &lockingBuffer{} defer func() { if t.Failed() { t.Logf("containerboot output:\n%s", cbOut.String()) } }() cmd.Stderr = cbOut cmd.Stdout = cbOut if err := cmd.Start(); err != nil { t.Fatalf("starting containerboot: %v", err) } defer func() { cmd.Process.Signal(unix.SIGTERM) cmd.Process.Wait() }()
var wantCmds []string for i, p := range tc.Phases { for k, v := range p.UpdateKubeSecret { env.kube.SetSecret(k, v) } for path, content := range p.UpdateFiles { fullPath := filepath.Join(env.d, path) if err := os.WriteFile(fullPath, []byte(content), 0700); err != nil { t.Fatalf("phase %d: updating file %q: %v", i, path, err) } // Explicitly update mtime to ensure fsnotify detects the change. // Without this, file operations can be buffered and fsnotify events may not trigger. now := time.Now() if err := os.Chtimes(fullPath, now, now); err != nil { t.Fatalf("phase %d: updating mtime for %q: %v", i, path, err) } } env.lapi.Notify(p.Notify) if p.Signal != nil { cmd.Process.Signal(*p.Signal) }
if p.WantExitCode != nil { state, err := cmd.Process.Wait() if err != nil { t.Fatal(err) } if state.ExitCode() != *p.WantExitCode { t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode()) } }
if p.WantLog != "" { err := tstest.WaitFor(5*time.Second, func() error { waitLogLine(t, 5*time.Second, cbOut, p.WantLog) return nil }) if err != nil { t.Fatal(err) } }
err := tstest.WaitFor(5*time.Second, func() error { if p.WantKubeSecret != nil { got := env.kube.Secret() if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" { return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff) } } else { got := env.kube.Secret() if len(got) > 0 { return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got) } } return nil }) if err != nil { t.Fatalf("test: %q phase %d: %v", name, i, err) }
// if we provide a wanted exit code, we expect that the process is finished, // so should return from the test. if p.WantExitCode != nil { return }
wantCmds = append(wantCmds, p.WantCmds...) waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n"))
err = tstest.WaitFor(2*time.Second, func() error { for path, want := range p.WantFiles { gotBs, err := os.ReadFile(filepath.Join(env.d, path)) if err != nil { return fmt.Errorf("reading wanted file %q: %v", path, err) } if got := strings.TrimSpace(string(gotBs)); got != want { return fmt.Errorf("wrong file contents for %q, got %q want %q", path, got, want) } } return nil }) if err != nil { t.Fatalf("phase %d: %v", i, err) }
for url, want := range p.EndpointStatuses { err := tstest.WaitFor(2*time.Second, func() error { resp, err := http.Get(url) if err != nil && want != -1 { return fmt.Errorf("GET %s: %v", url, err) } if want > 0 && resp.StatusCode != want { defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) return fmt.Errorf("GET %s, want %d, got %d\n%s", url, want, resp.StatusCode, string(body)) }
return nil }) if err != nil { t.Fatalf("phase %d: %v", i, err) } } } waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal") if cmd.ProcessState != nil { t.Fatalf("containerboot should be running but exited with exit code %d", cmd.ProcessState.ExitCode()) } }) }}
type lockingBuffer struct { sync.Mutex b bytes.Buffer}
func (b *lockingBuffer) Write(bs []byte) (int, error) { b.Lock() defer b.Unlock() return b.b.Write(bs)}
func (b *lockingBuffer) String() string { b.Lock() defer b.Unlock() return b.b.String()}
// waitLogLine looks for want in the contents of b.//// Only lines starting with 'boot: ' (the output of containerboot// itself) are considered, and the logged timestamp is ignored.//// waitLogLine fails the entire test if path doesn't contain want// before the timeout.func waitLogLine(t *testing.T, timeout time.Duration, b *lockingBuffer, want string) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { for line := range strings.SplitSeq(b.String(), "\n") { if !strings.HasPrefix(line, "boot: ") { continue } if strings.HasSuffix(line, " "+want) { return } } time.Sleep(100 * time.Millisecond) } t.Fatalf("timed out waiting for wanted output line %q. Output:\n%s", want, b.String())}
// waitArgs waits until the contents of path matches wantArgs, a set// of command lines recorded by test_tailscale.sh and// test_tailscaled.sh.//// All occurrences of removeStr are removed from the file prior to// comparison. This is used to remove the varying temporary root// directory name from recorded commandlines, so that wantArgs can be// a constant value.//// waitArgs fails the entire test if path doesn't contain wantArgs// before the timeout.func waitArgs(t *testing.T, timeout time.Duration, removeStr, path, wantArgs string) { t.Helper() wantArgs = strings.TrimSpace(wantArgs) deadline := time.Now().Add(timeout) var got string for time.Now().Before(deadline) { bs, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { // Don't bother logging that the file doesn't exist, it // should start existing soon. goto loop } else if err != nil { t.Logf("reading %q: %v", path, err) goto loop } got = strings.TrimSpace(string(bs)) got = strings.ReplaceAll(got, removeStr, "") if got == wantArgs { return } loop: time.Sleep(100 * time.Millisecond) } t.Fatalf("waiting for args file %q to have expected output, got:\n%s\n\nWant: %s", path, got, wantArgs)}
//go:embed test_tailscaled.shvar fakeTailscaled []byte
//go:embed test_tailscale.shvar fakeTailscale []byte
// localAPI is a minimal fake tailscaled LocalAPI server that presents// just enough functionality for containerboot to function// correctly. In practice this means it only supports querying// tailscaled status, and panics on all other uses to make it very// obvious that something unexpected happened.type localAPI struct { FSRoot string Path string // populated by Start
srv *http.Server
sync.Mutex cond *sync.Cond notify *ipn.Notify status *ipnstate.Status}
func (lc *localAPI) Start() error { path := filepath.Join(lc.FSRoot, "tmp/tailscaled.sock.fake") if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return err }
ln, err := net.Listen("unix", path) if err != nil { return err }
lc.srv = &http.Server{ Handler: lc, } lc.Path = path lc.cond = sync.NewCond(&lc.Mutex) go lc.srv.Serve(ln) return nil}
func (lc *localAPI) Close() { lc.srv.Close()}
func (lc *localAPI) Notify(n *ipn.Notify) { if n == nil { return } lc.Lock() defer lc.Unlock() lc.notify = n if n.InitialStatus != nil { lc.status = n.InitialStatus } else if lc.status == nil { lc.status = statusFromNotify(n) } else { if n.State != nil { lc.status.BackendState = n.State.String() } if n.SelfChange != nil { lc.status.Self = peerStatusFromNode(n.SelfChange.View()) } for _, p := range n.PeersChanged { if lc.status.Peer == nil { lc.status.Peer = map[key.NodePublic]*ipnstate.PeerStatus{} } pv := p.View() lc.status.Peer[pv.Key()] = peerStatusFromNode(pv) } for _, id := range n.PeersRemoved { for k, p := range lc.status.Peer { if p.NodeID == id { delete(lc.status.Peer, k) } } } } lc.cond.Broadcast()}
func statusFromNotify(n *ipn.Notify) *ipnstate.Status { st := new(ipnstate.Status) if n.State != nil { st.BackendState = n.State.String() } if n.SelfChange != nil { st.Self = peerStatusFromNode(n.SelfChange.View()) } if len(n.PeersChanged) != 0 { st.Peer = map[key.NodePublic]*ipnstate.PeerStatus{} for _, p := range n.PeersChanged { pv := p.View() st.Peer[pv.Key()] = peerStatusFromNode(pv) } } return st}
func peerStatusFromNode(n tailcfg.NodeView) *ipnstate.PeerStatus { ps := &ipnstate.PeerStatus{ ID: n.StableID(), NodeID: n.ID(), PublicKey: n.Key(), DNSName: n.Name(), } for _, p := range n.Addresses().All() { if p.IsSingleIP() { ps.TailscaleIPs = append(ps.TailscaleIPs, p.Addr()) } } if n.AllowedIPs().Len() != 0 { v := n.AllowedIPs() ps.AllowedIPs = &v } return ps}
func (lc *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/localapi/v0/serve-config": switch r.Method { case "GET": w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(&ipn.ServeConfig{}) return case "POST": return default: panic(fmt.Sprintf("unsupported method %q", r.Method)) } case "/localapi/v0/watch-ipn-bus": if r.Method != "GET" { panic(fmt.Sprintf("unsupported method %q", r.Method)) } case "/localapi/v0/usermetrics": if r.Method != "GET" { panic(fmt.Sprintf("unsupported method %q", r.Method)) } w.Write([]byte("fake metrics")) return case "/localapi/v0/prefs": switch r.Method { case "GET": w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(&ipn.Prefs{}) return case "PATCH": // EditPrefs - just return empty prefs w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(&ipn.Prefs{}) return default: panic(fmt.Sprintf("unsupported method %q", r.Method)) } // In the localAPI ServeHTTP method case "/localapi/v0/disconnect-control": if r.Method != "POST" { panic(fmt.Sprintf("unsupported method %q", r.Method)) } w.WriteHeader(http.StatusOK) return default: panic(fmt.Sprintf("unsupported path %q", r.URL.Path)) }
w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) if f, ok := w.(http.Flusher); ok { f.Flush() } enc := json.NewEncoder(w) lc.Lock() defer lc.Unlock() first := true for { if lc.notify != nil { n := *lc.notify if first { n.InitialStatus = lc.status first = false } if err := enc.Encode(&n); err != nil { // Usually broken pipe as the test client disconnects. return } if f, ok := w.(http.Flusher); ok { f.Flush() } } lc.cond.Wait() }}
// kubeServer is a minimal fake Kubernetes server that presents just// enough functionality for containerboot to function correctly. In// practice this means it only supports reading and modifying a single// kube secret, and panics on all other uses to make it very obvious// that something unexpected happened.type kubeServer struct { FSRoot string Host, Port string // populated by Start
srv *httptest.Server
sync.Mutex secret map[string]string canPatch bool}
func (k *kubeServer) Secret() map[string]string { k.Lock() defer k.Unlock() ret := map[string]string{} maps.Copy(ret, k.secret) return ret}
func (k *kubeServer) SetSecret(key, val string) { k.Lock() defer k.Unlock() k.secret[key] = val}
func (k *kubeServer) SetPatching(canPatch bool) { k.Lock() defer k.Unlock() k.canPatch = canPatch}
func (k *kubeServer) Start(t *testing.T) { k.secret = map[string]string{} root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount")
if err := os.MkdirAll(root, 0700); err != nil { t.Fatal(err) }
if err := os.WriteFile(filepath.Join(root, "namespace"), []byte("default"), 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "token"), []byte("bearer_token"), 0600); err != nil { t.Fatal(err) }
k.srv = httptest.NewTLSServer(k) k.Host = k.srv.Listener.Addr().(*net.TCPAddr).IP.String() k.Port = strconv.Itoa(k.srv.Listener.Addr().(*net.TCPAddr).Port)
var cert bytes.Buffer if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: k.srv.Certificate().Raw}); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "ca.crt"), cert.Bytes(), 0600); err != nil { t.Fatal(err) }}
func (k *kubeServer) Close() { k.srv.Close()}
func (k *kubeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer bearer_token" { panic("client didn't provide bearer token in request") } switch r.URL.Path { case "/api/v1/namespaces/default/secrets/tailscale": k.serveSecret(w, r) case "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews": k.serveSSAR(w, r) default: panic(fmt.Sprintf("unhandled fake kube api path %q", r.URL.Path)) }}
func (k *kubeServer) serveSSAR(w http.ResponseWriter, r *http.Request) { var req struct { Spec struct { ResourceAttributes struct { Verb string `json:"verb"` } `json:"resourceAttributes"` } `json:"spec"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { panic(fmt.Sprintf("decoding SSAR request: %v", err)) } ok := true if req.Spec.ResourceAttributes.Verb == "patch" { k.Lock() defer k.Unlock() ok = k.canPatch } // Just say yes to all SARs, we don't enforce RBAC. w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `{"status":{"allowed":%v}}`, ok)}
func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { bs, err := io.ReadAll(r.Body) if err != nil { http.Error(w, fmt.Sprintf("reading request body: %v", err), http.StatusInternalServerError) return } defer r.Body.Close()
switch r.Method { case "GET": w.Header().Set("Content-Type", "application/json") ret := map[string]map[string]string{ "data": {}, } k.Lock() defer k.Unlock() for k, v := range k.secret { v := base64.StdEncoding.EncodeToString([]byte(v)) ret["data"][k] = v } if err := json.NewEncoder(w).Encode(ret); err != nil { panic("encode failed") } case "PATCH": k.Lock() defer k.Unlock() if !k.canPatch { panic("containerboot tried to patch despite not being allowed") } switch r.Header.Get("Content-Type") { case "application/json-patch+json": req := []kubeclient.JSONPatch{} if err := json.Unmarshal(bs, &req); err != nil { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for _, op := range req { switch op.Op { case "remove": if !strings.HasPrefix(op.Path, "/data/") { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) case "add", "replace": path, ok := strings.CutPrefix(op.Path, "/data/") if !ok { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } val, ok := op.Value.(string) if !ok { panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", op.Value)) } v, err := base64.StdEncoding.DecodeString(val) if err != nil { panic(fmt.Sprintf("json patch value %q is not base64 encoded: %v", val, err)) } k.secret[path] = string(v) default: panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } } case "application/strategic-merge-patch+json": req := struct { Data map[string][]byte `json:"data"` }{} if err := json.Unmarshal(bs, &req); err != nil { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for key, val := range req.Data { if val == nil { delete(k.secret, key) } else { k.secret[key] = string(val) } } default: panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type"))) } default: panic(fmt.Sprintf("unhandled HTTP request %s %s", r.Method, r.URL)) }}
func mustJSON(t *testing.T, v any) []byte { b, err := json.Marshal(v) if err != nil { t.Fatalf("error converting %v to json: %v", v, err) } return b}
// egress services status given one named tailnet target specified by FQDN. As written by the proxy to its state Secret.func egressSvcStatus(name, fqdn, ip string) egressservices.Status { return egressservices.Status{ Services: map[string]*egressservices.ServiceStatus{ name: { TailnetTarget: egressservices.TailnetTarget{ FQDN: fqdn, }, TailnetTargetIPs: []netip.Addr{netip.MustParseAddr(ip)}, }, }, }}
// egress config given one named tailnet target specified by FQDN.func egressSvcConfig(name, fqdn string) egressservices.Configs { return egressservices.Configs{ name: egressservices.Config{ TailnetTarget: egressservices.TailnetTarget{ FQDN: fqdn, }, }, }}
// testEnv represents the environment needed for a single sub-test so that tests// can run in parallel.type testEnv struct { kube *kubeServer // Fake kube server. lapi *localAPI // Local TS API server. d string // Temp dir for the specific test. argFile string // File with commands test_tailscale{,d}.sh were invoked with. runningSockPath string // Path to the running tailscaled socket. localAddrPort int // Port for the containerboot HTTP server. healthAddrPort int // Port for the (deprecated) containerboot health server.}
func newTestEnv(t *testing.T) testEnv { d := t.TempDir()
lapi := localAPI{FSRoot: d} if err := lapi.Start(); err != nil { t.Fatal(err) } t.Cleanup(lapi.Close)
kube := kubeServer{FSRoot: d} kube.Start(t) t.Cleanup(kube.Close)
tailscaledConf := &ipn.ConfigVAlpha{AuthKey: new(configFileAuthKey), Version: "alpha0"} serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} serveConfWithServices := ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ "svc:test-service-1": {}, "svc:test-service-2": {}, }, } egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net")
dirs := []string{ "var/lib", "usr/bin", "tmp", "dev/net", "proc/sys/net/ipv4", "proc/sys/net/ipv6/conf/all", "etc/tailscaled", } for _, path := range dirs { if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil { t.Fatal(err) } } files := map[string][]byte{ "usr/bin/tailscaled": fakeTailscaled, "usr/bin/tailscale": fakeTailscale, "usr/bin/iptables": fakeTailscale, "usr/bin/ip6tables": fakeTailscale, "dev/net/tun": []byte(""), "proc/sys/net/ipv4/ip_forward": []byte("0"), "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), "etc/tailscaled/serve-config-with-services.json": mustJSON(t, serveConfWithServices), filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), } for path, content := range files { // Making everything executable is a little weird, but the // stuff that doesn't need to be executable doesn't care if we // do make it executable. if err := os.WriteFile(filepath.Join(d, path), content, 0700); err != nil { t.Fatal(err) } }
argFile := filepath.Join(d, "args") runningSockPath := filepath.Join(d, "tmp/tailscaled.sock") var localAddrPort, healthAddrPort int for _, p := range []*int{&localAddrPort, &healthAddrPort} { ln, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("Failed to open listener: %v", err) } if err := ln.Close(); err != nil { t.Fatalf("Failed to close listener: %v", err) } port := ln.Addr().(*net.TCPAddr).Port *p = port }
return testEnv{ kube: &kube, lapi: &lapi, d: d, argFile: argFile, runningSockPath: runningSockPath, localAddrPort: localAddrPort, healthAddrPort: healthAddrPort, }}
// TestProcessNotifyRefreshesDNSOnSelfChange verifies that a SelfChange// notification triggers a DNS refresh; without it, VIPServices created// after pod boot are invisible to resolveTailnetFQDN.func TestProcessNotifyRefreshesDNSOnSelfChange(t *testing.T) { extraRec := tailcfg.DNSRecord{ Name: "my-ingress.tailnet.ts.net.", Type: "A", Value: "100.99.10.20", } dnsCfg := &tailcfg.DNSConfig{ ExtraRecords: []tailcfg.DNSRecord{extraRec}, CertDomains: []string{"node.tailnet.ts.net"}, }
lal := memnet.Listen("local-tailscaled.sock:80") defer lal.Close() mux := http.NewServeMux() mux.HandleFunc("/localapi/v0/dns-config", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(dnsCfg); err != nil { t.Errorf("encoding dns config: %v", err) } }) srv := &http.Server{Handler: mux} go srv.Serve(lal) t.Cleanup(func() { srv.Shutdown(context.Background()) })
client := &local.Client{Dial: lal.Dial}
// Empty starting state, as if the InitialStatus captured at pod // boot carried no ExtraRecords because the VIPService didn't exist // yet at that time. var s netmapState
n := ipn.Notify{ SelfChange: &tailcfg.Node{ ID: 1, Name: "self.tailnet.ts.net.", }, }
got := s.processNotify(context.Background(), client, n)
if got.dnsExtraRecords.Len() != 1 { t.Fatalf("dnsExtraRecords.Len() = %d, want 1", got.dnsExtraRecords.Len()) } if rec := got.dnsExtraRecords.At(0); rec.Name != extraRec.Name { t.Errorf("dnsExtraRecords[0].Name = %q, want %q", rec.Name, extraRec.Name) } if got.certDomains.Len() != 1 || got.certDomains.At(0) != "node.tailnet.ts.net" { t.Errorf("certDomains = %v, want [node.tailnet.ts.net]", got.certDomains.AsSlice()) }}
func TestContainerbootWatchMask(t *testing.T) { const base = ipn.NotifyInitialStatus | ipn.NotifyNoNetMap tests := []struct { name string cfg settings want ipn.NotifyWatchOpt }{ {name: "subnet_router", cfg: settings{Routes: new("10.0.0.0/8")}, want: base}, {name: "tailnet_target_IP", cfg: settings{TailnetTargetIP: "100.64.0.1"}, want: base}, {name: "tailnet_target_FQDN", cfg: settings{TailnetTargetFQDN: "target.example.ts.net"}, want: base | ipn.NotifyPeerChanges}, {name: "egress_services", cfg: settings{EgressProxiesCfgPath: "/etc/egress-services"}, want: base | ipn.NotifyPeerChanges}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := containerbootWatchMask(&tt.cfg); got != tt.want { t.Errorf("containerbootWatchMask() = %v, want %v", got, tt.want) } }) }}
func TestUpdateFromStatusReplacesState(t *testing.T) { oldPeer := &tailcfg.Node{ID: 1, StableID: "old", Name: "old.example.ts.net."} s := netmapState{ self: oldPeer.View(), certDomains: views.SliceOf([]string{"old.example.ts.net"}), dnsExtraRecords: views.SliceOf([]tailcfg.DNSRecord{{Name: "old.example.ts.net."}}), }.upsertPeer(oldPeer.View())
got := s.updateFromStatus(new(ipnstate.Status)) if got.self.Valid() { t.Error("self remained valid after an empty authoritative status") } if got.peersByID != nil && got.peersByID.Len() != 0 { t.Errorf("peer count = %d, want 0", got.peersByID.Len()) } if got.certDomains.Len() != 0 || got.dnsExtraRecords.Len() != 0 { t.Error("DNS state remained after an empty authoritative status") }}
type scriptedIPNBusWatcher struct { notifies []ipn.Notify err error}
func (w *scriptedIPNBusWatcher) Close() error { return nil }
func (w *scriptedIPNBusWatcher) Next() (ipn.Notify, error) { if len(w.notifies) == 0 { return ipn.Notify{}, w.err } n := w.notifies[0] w.notifies = w.notifies[1:] return n, nil}
func TestReconnectingIPNBusWatcher(t *testing.T) { terminalMessage := "IPN bus consumer fell behind; closing watch" watches := []*scriptedIPNBusWatcher{ {notifies: []ipn.Notify{{ErrMessage: &terminalMessage}}, err: io.EOF}, {notifies: []ipn.Notify{{InitialStatus: &ipnstate.Status{BackendState: ipn.Running.String()}}}, err: io.EOF}, } var ( mu sync.Mutex masks []ipn.NotifyWatchOpt ) ctx, cancel := context.WithCancel(t.Context()) defer cancel() watch := func(_ context.Context, mask ipn.NotifyWatchOpt) (klc.IPNBusWatcher, error) { mu.Lock() defer mu.Unlock() masks = append(masks, mask) if len(watches) == 0 { return nil, errors.New("no scripted watcher") } w := watches[0] watches = watches[1:] return w, nil } mask := ipn.NotifyInitialStatus | ipn.NotifyNoNetMap w := newReconnectingIPNBusWatcher(ctx, "test", watch, mask, time.Millisecond) defer w.Close()
n, err := w.Next() if err != nil { t.Fatal(err) } if n.ErrMessage == nil || *n.ErrMessage != terminalMessage { t.Fatalf("first notification = %+v, want terminal error", n) }
n, err = w.Next() if err != nil { t.Fatal(err) } if n.InitialStatus == nil || n.InitialStatus.BackendState != ipn.Running.String() { t.Fatalf("notification = %+v, want replacement running status", n) } cancel() if _, err := w.Next(); !errors.Is(err, context.Canceled) { t.Fatalf("Next error = %v, want context.Canceled", err) }
mu.Lock() defer mu.Unlock() if len(masks) < 2 { t.Fatalf("watch attempts = %d, want at least 2", len(masks)) } for i, got := range masks[:2] { if got != mask { t.Errorf("watch attempt %d mask = %v, want %v", i, got, mask) } }}
func TestReconnectingIPNBusWatcherGivesUpOnDialFailure(t *testing.T) { dialErr := errors.New("connection refused") var attempts int watch := func(context.Context, ipn.NotifyWatchOpt) (klc.IPNBusWatcher, error) { attempts++ return nil, dialErr } w := newReconnectingIPNBusWatcher(t.Context(), "test", watch, ipn.NotifyInitialStatus, time.Millisecond) w.maxDialFailure = 20 * time.Millisecond
if _, err := w.Next(); !errors.Is(err, dialErr) { t.Fatalf("Next error = %v, want %v", err, dialErr) } if attempts < 2 { t.Errorf("watch attempts = %d, want at least 2", attempts) }}