diff --git a/Gopkg.lock b/Gopkg.lock index 593c0257f..6fe9b4cdc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -35,7 +35,7 @@ revision = "de5bf2ad457846296e2031421a34e2568e304e35" [[projects]] - digest = "1:35768db37daed3f5e0535a68e7dba84dc60ad2a8cedb371e4862b25fc043c243" + digest = "1:8404a965fb2ea6fb9f8b23d738b7abbc4e167483a5289e1bd02520f91651a866" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -47,6 +47,7 @@ "aws/credentials", "aws/credentials/ec2rolecreds", "aws/credentials/endpointcreds", + "aws/credentials/processcreds", "aws/credentials/stscreds", "aws/csm", "aws/defaults", @@ -73,8 +74,8 @@ "service/sts", ] pruneopts = "UT" - revision = "ceab7b7ac6f535d1397f124620720b5145f1ad59" - version = "v1.15.88" + revision = "a2484497433aa110263c045df2f1d49336b38cfd" + version = "v1.16.22" [[projects]] branch = "master" @@ -85,7 +86,7 @@ revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] - digest = "1:4f618dd2213260a992922925a03349430af6994d8a481c53280703732e5bc4ab" + digest = "1:50f39e37fd7347733d97e4a88a3d4674ead4abc2eab15b25522f5d9dd5495dd3" name = "github.com/coreos/etcd" packages = [ "auth/authpb", @@ -102,8 +103,8 @@ "version", ] pruneopts = "UT" - revision = "27fc7e2296f506182f58ce846e48f36b34fe6842" - version = "v3.3.10" + revision = "2cf9e51d2a78003b164c2998886158e60ded1cbb" + version = "v3.3.11" [[projects]] digest = "1:0ef770954bca104ee99b3b6b7f9b240605ac03517d9f98cbc1893daa03f3c038" @@ -167,15 +168,15 @@ version = "v1.0.0" [[projects]] - digest = "1:899234af23e5793c34e06fd397f86ba33af5307b959b6a7afd19b63db065a9d7" + digest = "1:9f6d41e901cde897bad4569877c96c9cacee67c9c6ede2794c515371c954cb6a" name = "github.com/emicklei/go-restful" packages = [ ".", "log", ] pruneopts = "UT" - revision = "3eb9738c1697594ea6e71a7156a9bb32ed216cf0" - version = "v2.8.0" + revision = "e37671aced663c8d3a395bd301857e26dcf4340c" + version = "v2.8.1" [[projects]] digest = "1:ddab18e89cf46e40707b89dbe3835b4a591b0ea298e1035eefa84002aa9a4b4e" @@ -210,12 +211,12 @@ version = "v1.0.0" [[projects]] - digest = "1:15e27372d379b45b18ac917b9dafc45c45485239490ece18cca97a12f9591146" + digest = "1:8b4bb15de2e698d359eeec15c42453567c98972a70c8619946050360ec02ec32" name = "github.com/go-ini/ini" packages = ["."] pruneopts = "UT" - revision = "9c8236e659b76e87bf02044d06fde8683008ff3e" - version = "v1.39.0" + revision = "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe" + version = "v1.41.0" [[projects]] digest = "1:953a2628e4c5c72856b53f5470ed5e071c55eccf943d798d42908102af2a610f" @@ -223,7 +224,7 @@ packages = ["."] pruneopts = "UT" revision = "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" - version = "v0.17.2" + version = "v0.18.0" [[projects]] digest = "1:81210e0af657a0fb3638932ec68e645236bceefa4c839823db0c4d918f080895" @@ -231,23 +232,23 @@ packages = ["."] pruneopts = "UT" revision = "8483a886a90412cd6858df4ea3483dce9c8e35a3" - version = "v0.17.2" + version = "v0.18.0" [[projects]] - digest = "1:394fed5c0425fe01da3a34078adaa1682e4deaea6e5d232dde25c4034004c151" + digest = "1:08656ef9c5a45ddccb7f206ca2d67e12e9fcda4122a83dc0544b5c967267cefa" name = "github.com/go-openapi/spec" packages = ["."] pruneopts = "UT" - revision = "5bae59e25b21498baea7f9d46e9c147ec106a42e" - version = "v0.17.2" + revision = "5b6cdde3200976e3ecceb2868706ee39b6aff3e4" + version = "v0.18.0" [[projects]] - digest = "1:32f3d2e7f343de7263c550d696fb8a64d3c49d8df16b1ddfc8e80e1e4b3233ce" + digest = "1:0005186c6608dd542239ac8e4f4f1e2e7c24d493e999113c46b93332f0362fc0" name = "github.com/go-openapi/swag" packages = ["."] pruneopts = "UT" - revision = "5899d5c5e619fda5fa86e14795a835f473ca284c" - version = "v0.17.2" + revision = "1d29f06aebd59ccdf11ae04aa0334ded96e2d909" + version = "v0.18.0" [[projects]] digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65" @@ -258,7 +259,7 @@ version = "v1.4.1" [[projects]] - digest = "1:db238461f652ddb7c7057bc6fc503f6003a29987b1485ecbb96d92287db65bc9" + digest = "1:1f4955627f01f4168c991891aae53ea7c0af139cf2d66bd641978a7bd50da841" name = "github.com/gogo/protobuf" packages = [ "gogoproto", @@ -267,8 +268,8 @@ "sortkeys", ] pruneopts = "UT" - revision = "636bf0302bc95575d69441b25a2603156ffdddf1" - version = "v1.1.1" + revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7" + version = "v1.2.0" [[projects]] branch = "master" @@ -348,12 +349,20 @@ revision = "c63ab54fda8f77302f8d414e19933f2b6026a089" [[projects]] - digest = "1:77395dd3847dac9c45118c668f5dab85aedf0163dc3b38aea6578c5cf0d502f9" + digest = "1:9b7a07ac7577787a8ecc1334cb9f34df1c76ed82a917d556c5713d3ab84fbc43" + name = "github.com/grpc-ecosystem/go-grpc-prometheus" + packages = ["."] + pruneopts = "UT" + revision = "c225b8c3b01faf2899099b768856a9e916e5087b" + version = "v1.2.0" + +[[projects]] + digest = "1:950caca7dfcf796419232ba996c9c3539d09f26af27ba848c4508e604c13efbb" name = "github.com/hashicorp/go-version" packages = ["."] pruneopts = "UT" - revision = "b5a281d3160aa11950a6182bd9a9dc2cb1e02d50" - version = "v1.0.0" + revision = "d40cf49b3a77bba84a7afdbd7f1dc295d114efb1" + version = "v1.1.0" [[projects]] digest = "1:8ec8d88c248041a6df5f6574b87bc00e7e0b493881dad2e7ef47b11dc69093b5" @@ -399,11 +408,11 @@ revision = "82859aed1b5dafbbe6ad4f3dac5404d2a742cf18" [[projects]] - digest = "1:e22af8c7518e1eab6f2eab2b7d7558927f816262586cd6ed9f349c97a6c285c4" + digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" name = "github.com/jmespath/go-jmespath" packages = ["."] pruneopts = "UT" - revision = "0b12d6b5" + revision = "c2b33e84" [[projects]] digest = "1:3e551bbb3a7c0ab2a2bf4660e7fcad16db089fdcfbb44b0199e62838038623ea" @@ -413,6 +422,18 @@ revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" +[[projects]] + digest = "1:87b8c5f48e71daa0fa2e0f1f18fbaafc797bc287f4da3bedc5ed653f3443dc11" + name = "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner" + packages = [ + "controller", + "controller/metrics", + "util", + ] + pruneopts = "UT" + revision = "cdaf8fa71bb51d0a633d28d6f4ab46063d6e8f94" + version = "v1.0.0" + [[projects]] branch = "master" digest = "1:84a5a2b67486d5d67060ac393aa255d05d24ed5ee41daecd5635ec22657b6492" @@ -433,6 +454,14 @@ revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" +[[projects]] + digest = "1:d64a0aa67f715fd28f5ddfe0d74c06b6d6fe0fed931d77e662f2c4694f6854dc" + name = "github.com/miekg/dns" + packages = ["."] + pruneopts = "UT" + revision = "56be65265e34e731425e0269a301774938827c60" + version = "v1.1.3" + [[projects]] digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" @@ -449,6 +478,14 @@ revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" +[[projects]] + branch = "master" + digest = "1:24df057f15e7a09e75c1241cbe6f6590fd3eac9804f1110b02efade3214f042d" + name = "github.com/munnerz/goautoneg" + packages = ["."] + pruneopts = "UT" + revision = "a547fc61f48d567d5b4ec6f8aee5573d8efce11d" + [[projects]] digest = "1:ee4d4af67d93cc7644157882329023ce9a7bcfce956a079069a9405521c7cc8d" name = "github.com/opencontainers/go-digest" @@ -490,15 +527,16 @@ version = "v1.0.0" [[projects]] - digest = "1:735956b5e94182229dfbdc5b401d7a71f20f54f1259dbc2969ec2b56bfbb34f2" + digest = "1:93a746f1060a8acbcf69344862b2ceced80f854170e1caae089b2834c5fbf7f4" name = "github.com/prometheus/client_golang" packages = [ "prometheus", "prometheus/internal", + "prometheus/promhttp", ] pruneopts = "UT" - revision = "abad2d1bd44235a26707c172eab6bca5bf2dbad3" - version = "v0.9.1" + revision = "505eaef017263e299324067d40ca2c48f6a2cf50" + version = "v0.9.2" [[projects]] branch = "master" @@ -506,11 +544,11 @@ name = "github.com/prometheus/client_model" packages = ["go"] pruneopts = "UT" - revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + revision = "56726106282f1985ea77d5305743db7231b0c0a8" [[projects]] branch = "master" - digest = "1:db712fde5d12d6cdbdf14b777f0c230f4ff5ab0be8e35b239fc319953ed577a4" + digest = "1:ce62b400185bf6b16ef6088011b719e449f5c15c4adb6821589679f752c2788e" name = "github.com/prometheus/common" packages = [ "expfmt", @@ -518,11 +556,11 @@ "model", ] pruneopts = "UT" - revision = "4724e9255275ce38f7179b2478abeae4e28c904f" + revision = "2998b132700a7d019ff618c06a234b47c1f3f681" [[projects]] branch = "master" - digest = "1:9ea97158ea06d60b1f324aa384f6a4cbd596a81b3cae994e17108b195f398d01" + digest = "1:f532f2cdb9e9e4a8fad5a7e944482f4dd12650228e4a7a6c8492a0d069ce0690" name = "github.com/prometheus/procfs" packages = [ ".", @@ -531,7 +569,7 @@ "xfs", ] pruneopts = "UT" - revision = "aa55a523dc0a8297edf51bb75e8eec13eb3be45d" + revision = "bf6a532e95b1f7a62adf0ab5050a5bb2237ad2f4" [[projects]] branch = "master" @@ -542,15 +580,15 @@ revision = "3a90049568e908f071d810b17fa3dfcf2b78b013" [[projects]] - digest = "1:6a4a11ba764a56d2758899ec6f3848d24698d48442ebce85ee7a3f63284526cd" + digest = "1:d707dbc1330c0ed177d4642d6ae102d5e2c847ebd0eb84562d0dc4f024531cfc" name = "github.com/spf13/afero" packages = [ ".", "mem", ] pruneopts = "UT" - revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd" - version = "v1.1.2" + revision = "a5d6946387efe7d64d09dcba68cdd523dc1273a3" + version = "v1.2.0" [[projects]] digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" @@ -569,7 +607,7 @@ version = "v1.0.3" [[projects]] - digest = "1:5110e3d4f130772fd39e6ce8208ad1955b242ccfcc8ad9d158857250579c82f4" + digest = "1:8ff03ccc603abb0d7cce94d34b613f5f6251a9e1931eba1a3f9888a9029b055c" name = "github.com/stretchr/testify" packages = [ "assert", @@ -577,8 +615,8 @@ "suite", ] pruneopts = "UT" - revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" - version = "v1.2.2" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" [[projects]] digest = "1:03aa6e485e528acb119fb32901cf99582c380225fc7d5a02758e08b180cb56c3" @@ -611,35 +649,52 @@ "ssh/terminal", ] pruneopts = "UT" - revision = "eb0de9b17e854e9b1ccd9963efafc79862359959" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" - digest = "1:b69ed3091785699803ec77e55e9d9f9397c3ca0d821e14a7d6e2f9db105592a1" + digest = "1:fddc5d9d26971eb4ea4a4f3f92cc31557d19c30016d1601dd17dbaa9051f304d" name = "golang.org/x/net" packages = [ + "bpf", "context", + "context/ctxhttp", "http/httpguts", "http2", "http2/hpack", "idna", + "internal/iana", + "internal/socket", "internal/timeseries", + "ipv4", + "ipv6", "trace", "websocket", ] pruneopts = "UT" - revision = "fae4c4e3ad76c295c3d6d259f898136b4bf833a8" + revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" [[projects]] branch = "master" - digest = "1:0c7cb58af944f9690af91474a75371eecfe54c1ce3771a8dc08924430ee785d3" + digest = "1:e007b54f54cbd4214aa6d97a67d57bc2539991adb4e22ea92c482bbece8de469" + name = "golang.org/x/oauth2" + packages = [ + ".", + "internal", + ] + pruneopts = "UT" + revision = "5dab4167f31cbd76b407f1486c86b40748bc5073" + +[[projects]] + branch = "master" + digest = "1:43cde116ff48f299eddb7e6515677e6d0a2c915854bb05a333877f07c3bb3033" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "UT" - revision = "4ed8d59d0b35e1e29334a206d1b3f38b1e5dfb31" + revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" [[projects]] digest = "1:0c56024909189aee3364b7f21a95a27459f718aa7c199a5c111c36cfffd9eaef" @@ -675,53 +730,74 @@ [[projects]] branch = "master" - digest = "1:6511064c08bff306dd24fa83ea82e160009fa7572240b70173b25d1dbc3e1872" + digest = "1:3ab7ad4d64d2f8bd7dc7f4847f546a6f97eccba24be5c3b9c4618d6e2b080ac9" name = "golang.org/x/tools" packages = [ "go/ast/astutil", + "go/gcexportdata", + "go/internal/cgo", + "go/internal/gcimporter", + "go/internal/packagesdriver", + "go/packages", + "go/types/typeutil", "imports", "internal/fastwalk", "internal/gopathwalk", + "internal/semver", ] pruneopts = "UT" - revision = "1c3d964395ce8f04f3b03b30aaed0b096c08c3c6" + revision = "16909d206f00da7d0d5ba28cd9dc7fb223648ecf" [[projects]] - digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" + digest = "1:9e29a0ec029d012437d88da3ccccf18adcdce069cab08d462056c2c6bb006505" name = "google.golang.org/appengine" - packages = ["cloudsql"] + packages = [ + "cloudsql", + "internal", + "internal/base", + "internal/datastore", + "internal/log", + "internal/remote_api", + "internal/urlfetch", + "urlfetch", + ] pruneopts = "UT" - revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" - version = "v1.3.0" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] branch = "master" - digest = "1:56b0bca90b7e5d1facf5fbdacba23e4e0ce069d25381b8e2f70ef1e7ebfb9c1a" + digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] pruneopts = "UT" - revision = "31ac5d88444a9e7ad18077db9a165d793ad06a2e" + revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a" [[projects]] - digest = "1:bf7e2328a58176b7f29f3b425b0555745e9a18ed803f5fd8f6c2c13913a60bb4" + digest = "1:6dfe7f3314a390dc9e21368dd41236169bf40ae69674f42b7bd45db537751a94" name = "google.golang.org/grpc" packages = [ ".", "balancer", "balancer/base", "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", "codes", "connectivity", "credentials", + "credentials/internal", "encoding", "encoding/proto", "grpclog", "health/grpc_health_v1", "internal", "internal/backoff", + "internal/binarylog", "internal/channelz", "internal/envconfig", "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", "internal/transport", "keepalive", "metadata", @@ -735,8 +811,8 @@ "tap", ] pruneopts = "UT" - revision = "2e463a05d100327ca47ac218281906921038fd95" - version = "v1.16.0" + revision = "a02b0774206b209466313a0b525d2c738fe407eb" + version = "v1.18.0" [[projects]] digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" @@ -747,7 +823,7 @@ version = "v0.9.1" [[projects]] - digest = "1:ee5a2fc2456c13fcbd59b442b4a26687ff6bcd631b54412f0c52b6a16f78743f" + digest = "1:a4cde1eec9a17eb2399a50c6e1a9fe3fde039994de058f9dbf6592d157bfe97b" name = "gopkg.in/square/go-jose.v2" packages = [ ".", @@ -756,8 +832,8 @@ "jwt", ] pruneopts = "UT" - revision = "f61ac651e23224d43848a8dc387dee016d7eacf9" - version = "v2.2.0" + revision = "e94fb177d3668d35ab39c61cbb2f311550557e83" + version = "v2.2.2" [[projects]] digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" @@ -768,7 +844,7 @@ version = "v2.2.2" [[projects]] - digest = "1:13f16734c26d843bc2accbc21dd49be71de99ad5fa0176726cdada80cc6859cb" + digest = "1:5a6800974eb644ef203fef871e6ac74448e93f3c81a2ec1e10f4c3b17f37b99c" name = "k8s.io/api" packages = [ "admission/v1beta1", @@ -783,10 +859,12 @@ "authorization/v1beta1", "autoscaling/v1", "autoscaling/v2beta1", + "autoscaling/v2beta2", "batch/v1", "batch/v1beta1", "batch/v2alpha1", "certificates/v1beta1", + "coordination/v1beta1", "core/v1", "events/v1beta1", "extensions/v1beta1", @@ -803,11 +881,11 @@ "storage/v1beta1", ] pruneopts = "UT" - revision = "4e7be11eab3ffcfc1876898b8272df53785a9504" - version = "kubernetes-1.11.3" + revision = "fd83cbc87e7632ccd8bbab63d2b673d4e0c631cc" + version = "kubernetes-1.12.0" [[projects]] - digest = "1:415eb156615f37d8fd661ed078d29fa0a86f739b839b9da63b3d39d695e572c0" + digest = "1:68802413f6b2d016bc2a49fb2a66be5507bbfbae627f8b81badd30bac6c4aa26" name = "k8s.io/apiextensions-apiserver" packages = [ "pkg/apis/apiextensions", @@ -818,11 +896,11 @@ "pkg/features", ] pruneopts = "UT" - revision = "b05d9bb7cc74a62c5daac9cef31894343e5d2c9b" - version = "kubernetes-1.11.3" + revision = "1748dfb29e8a4432b78514bc88a1b07937a9805a" + version = "kubernetes-1.12.0" [[projects]] - digest = "1:3ac9d135ccf71b822fc64e13180a69a2a7df35c77327f731739619c41e8237a3" + digest = "1:df506fdedd1b6352ca65ed05a755163efe03dd96a91b612e7eeb2a71375ebbc7" name = "k8s.io/apimachinery" packages = [ "pkg/api/equality", @@ -831,6 +909,8 @@ "pkg/api/resource", "pkg/api/validation", "pkg/api/validation/path", + "pkg/apis/config", + "pkg/apis/config/v1alpha1", "pkg/apis/meta/internalversion", "pkg/apis/meta/v1", "pkg/apis/meta/v1/unstructured", @@ -858,6 +938,7 @@ "pkg/util/intstr", "pkg/util/json", "pkg/util/mergepatch", + "pkg/util/naming", "pkg/util/net", "pkg/util/rand", "pkg/util/runtime", @@ -875,11 +956,11 @@ "third_party/forked/golang/reflect", ] pruneopts = "UT" - revision = "def12e63c512da17043b4f0293f52d1006603d9f" - version = "kubernetes-1.11.3" + revision = "6dd46049f39503a1fc8d65de4bd566829e95faff" + version = "kubernetes-1.12.0" [[projects]] - digest = "1:2c4d91c9d73dee723a0f07d7e712f8366c7b939632b59c9002cbfdb7893f261d" + digest = "1:e7061d54d3959aeee13137bfaffda2d1488a82ce50ec07bdea60a862784d24b9" name = "k8s.io/apiserver" packages = [ "pkg/admission", @@ -897,14 +978,18 @@ "pkg/admission/plugin/webhook/namespace", "pkg/admission/plugin/webhook/request", "pkg/admission/plugin/webhook/rules", + "pkg/admission/plugin/webhook/util", "pkg/admission/plugin/webhook/validating", "pkg/apis/apiserver", "pkg/apis/apiserver/install", "pkg/apis/apiserver/v1alpha1", "pkg/apis/audit", + "pkg/apis/audit/v1", "pkg/apis/audit/v1alpha1", "pkg/apis/audit/v1beta1", "pkg/apis/audit/validation", + "pkg/apis/config", + "pkg/apis/config/v1alpha1", "pkg/audit", "pkg/audit/policy", "pkg/authentication/authenticator", @@ -951,8 +1036,10 @@ "pkg/storage/storagebackend", "pkg/storage/storagebackend/factory", "pkg/storage/value", + "pkg/util/dryrun", "pkg/util/feature", "pkg/util/flushwriter", + "pkg/util/logs", "pkg/util/openapi", "pkg/util/trace", "pkg/util/webhook", @@ -961,11 +1048,11 @@ "plugin/pkg/authorizer/webhook", ] pruneopts = "UT" - revision = "386115dd78fde3efc2366cb381420a4dd0157348" - version = "kubernetes-1.11.3" + revision = "e85ad7b666fef0476185731329f4cff1536efff8" + version = "kubernetes-1.12.0" [[projects]] - digest = "1:8f55e66a9679e0dc3d2c6211bfb60ec594a73461f23748b034b9a5940978ca22" + digest = "1:1b756ead87f036cff87cbeeeebc91101e69a7b6ba1589bd974957be242a8f3f3" name = "k8s.io/client-go" packages = [ "discovery", @@ -981,12 +1068,15 @@ "informers/autoscaling", "informers/autoscaling/v1", "informers/autoscaling/v2beta1", + "informers/autoscaling/v2beta2", "informers/batch", "informers/batch/v1", "informers/batch/v1beta1", "informers/batch/v2alpha1", "informers/certificates", "informers/certificates/v1beta1", + "informers/coordination", + "informers/coordination/v1beta1", "informers/core", "informers/core/v1", "informers/events", @@ -1036,6 +1126,8 @@ "kubernetes/typed/autoscaling/v1/fake", "kubernetes/typed/autoscaling/v2beta1", "kubernetes/typed/autoscaling/v2beta1/fake", + "kubernetes/typed/autoscaling/v2beta2", + "kubernetes/typed/autoscaling/v2beta2/fake", "kubernetes/typed/batch/v1", "kubernetes/typed/batch/v1/fake", "kubernetes/typed/batch/v1beta1", @@ -1044,6 +1136,8 @@ "kubernetes/typed/batch/v2alpha1/fake", "kubernetes/typed/certificates/v1beta1", "kubernetes/typed/certificates/v1beta1/fake", + "kubernetes/typed/coordination/v1beta1", + "kubernetes/typed/coordination/v1beta1/fake", "kubernetes/typed/core/v1", "kubernetes/typed/core/v1/fake", "kubernetes/typed/events/v1beta1", @@ -1079,10 +1173,12 @@ "listers/apps/v1beta2", "listers/autoscaling/v1", "listers/autoscaling/v2beta1", + "listers/autoscaling/v2beta2", "listers/batch/v1", "listers/batch/v1beta1", "listers/batch/v2alpha1", "listers/certificates/v1beta1", + "listers/coordination/v1beta1", "listers/core/v1", "listers/events/v1beta1", "listers/extensions/v1beta1", @@ -1107,16 +1203,17 @@ "testing", "tools/auth", "tools/cache", - "tools/cache/testing", "tools/clientcmd", "tools/clientcmd/api", "tools/clientcmd/api/latest", "tools/clientcmd/api/v1", + "tools/leaderelection", "tools/leaderelection/resourcelock", "tools/metrics", "tools/pager", "tools/record", "tools/reference", + "tools/watch", "transport", "util/buffer", "util/cert", @@ -1128,11 +1225,11 @@ "util/workqueue", ] pruneopts = "UT" - revision = "7d04d0e2a0a1a4d4a1cd6baa432a2301492e4e65" - version = "v8.0.0" + revision = "1638f8970cefaa404ff3a62950f88b08292b2696" + version = "v9.0.0" [[projects]] - digest = "1:3b9c82e73290987c719062ba514f47cc95a7b1e10d9868b1e083e48044c0820b" + digest = "1:4e2addcdbe0330f43800c1fcb905fc7a21b86415dfcca619e5c606c87257af1b" name = "k8s.io/code-generator" packages = [ "cmd/client-gen", @@ -1146,12 +1243,25 @@ "pkg/util", ] pruneopts = "T" - revision = "8c97d6ab64da020f8b151e9d3ed8af3172f5c390" - version = "kubernetes-1.11.3" + revision = "3dcf91f64f638563e5106f21f50c31fa361c918d" + version = "kubernetes-1.12.0" [[projects]] branch = "master" - digest = "1:c5636f57f83eb7ef2a1c7a2340b7e5750f1c105a1d0cbcfcab429d175c216773" + digest = "1:e79eeb85f6f9636d4a9f1c36f1bdab1b2684db3621d7feea1d087edab281154c" + name = "k8s.io/csi-api" + packages = [ + "pkg/apis/csi/v1alpha1", + "pkg/client/clientset/versioned", + "pkg/client/clientset/versioned/scheme", + "pkg/client/clientset/versioned/typed/csi/v1alpha1", + ] + pruneopts = "UT" + revision = "102ae30137c98787ea40b5eb9fe236496a2407fd" + +[[projects]] + branch = "master" + digest = "1:af1d397bf7cd0414aa2819ef68ea5bfca061dd4e035af41d6f17b0cecef91e2d" name = "k8s.io/gengo" packages = [ "args", @@ -1161,7 +1271,7 @@ "types", ] pruneopts = "T" - revision = "fd15ee9cc2f77baa4f31e59e6acbf21146455073" + revision = "f8a0810f38afb8478882b3835a615aebfda39afa" [[projects]] digest = "1:e2999bf1bb6eddc2a6aa03fe5e6629120a53088926520ca3b4765f77d7ff7eab" @@ -1171,9 +1281,17 @@ revision = "a5bc97fbc634d635061f3146511332c7e313a55a" version = "v0.1.0" +[[projects]] + digest = "1:bcd57f8e20b04c39d15eb74dc2708596f7e2b725c1df4282886f5f0f9b8ece18" + name = "k8s.io/kube-controller-manager" + packages = ["config/v1alpha1"] + pruneopts = "UT" + revision = "6b6338e3867d8d9697cf8de392d275943adaac75" + version = "kubernetes-1.12.0" + [[projects]] branch = "master" - digest = "1:1f066ea11b5de714e38e130805dd9a87097ee6df06fd75e89fdf1087bd611419" + digest = "1:287d9dc13d8ee76a4bd44e12da6e7422a1d111315b9d76d2887b31489bf7dc31" name = "k8s.io/kube-openapi" packages = [ "pkg/builder", @@ -1183,10 +1301,10 @@ "pkg/util/proto", ] pruneopts = "UT" - revision = "0317810137be915b9cf888946c6e115c1bfac693" + revision = "ced9eb3070a5f1c548ef46e8dfe2a97c208d9f03" [[projects]] - digest = "1:4c49ed4596ab078e17096f7cdf1d8253f4a562b3a7b9c613d992ef964fef2e84" + digest = "1:c7cbad5670a9fb12fd66c58800e2d14b83d4cef6256a6770d715db181495acdb" name = "k8s.io/kubernetes" packages = [ "pkg/api/legacyscheme", @@ -1194,7 +1312,6 @@ "pkg/api/v1/endpoints", "pkg/api/v1/pod", "pkg/apis/autoscaling", - "pkg/apis/componentconfig", "pkg/apis/core", "pkg/apis/core/helper", "pkg/apis/core/install", @@ -1225,16 +1342,14 @@ "pkg/serviceaccount", "pkg/util/file", "pkg/util/filesystem", - "pkg/util/goroutinemap", - "pkg/util/goroutinemap/exponentialbackoff", "pkg/util/hash", "pkg/util/io", "pkg/util/metrics", "pkg/util/mount", "pkg/util/net/sets", + "pkg/util/node", "pkg/util/nsenter", "pkg/util/parsers", - "pkg/util/pointer", "pkg/util/strings", "pkg/util/taints", "pkg/util/version", @@ -1247,16 +1362,19 @@ "pkg/volume/util/volumepathhandler", ] pruneopts = "UT" - revision = "a4529464e4629c21224b3d52edfe0ea91b072862" - version = "v1.11.3" + revision = "0ed33881dc4355495f623c6f22e7dd0b7632b7c0" + version = "v1.12.0" [[projects]] branch = "master" - digest = "1:381323c2fe2e890a3dd3b5d6dc6f2199068408cca89b24f6b7ca1c60f32644a5" + digest = "1:9c6cb46fa10409b199f93e6ceda462ddfa62d74e97174591a147b38982585d24" name = "k8s.io/utils" - packages = ["exec"] + packages = [ + "exec", + "pointer", + ] pruneopts = "UT" - revision = "0d26856f57b32ec3398579285e5c8a2bfe8c5243" + revision = "8a16e7dd8fb6d97d1331b0c79a16722f934b00b1" [solve-meta] analyzer-name = "dep" @@ -1275,6 +1393,7 @@ "github.com/google/uuid", "github.com/icrowley/fake", "github.com/jbw976/go-ps", + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller", "github.com/rook/operator-kit", "github.com/spf13/cobra", "github.com/spf13/pflag", @@ -1306,7 +1425,6 @@ "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/util/runtime", "k8s.io/apimachinery/pkg/util/strategicpatch", - "k8s.io/apimachinery/pkg/util/uuid", "k8s.io/apimachinery/pkg/util/validation", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/watch", @@ -1321,24 +1439,20 @@ "k8s.io/client-go/kubernetes/fake", "k8s.io/client-go/kubernetes/scheme", "k8s.io/client-go/kubernetes/typed/core/v1", - "k8s.io/client-go/kubernetes/typed/core/v1/fake", "k8s.io/client-go/listers/apps/v1", "k8s.io/client-go/listers/core/v1", "k8s.io/client-go/rest", "k8s.io/client-go/testing", "k8s.io/client-go/tools/cache", - "k8s.io/client-go/tools/cache/testing", "k8s.io/client-go/tools/record", - "k8s.io/client-go/tools/reference", "k8s.io/client-go/util/flowcontrol", "k8s.io/client-go/util/workqueue", "k8s.io/code-generator/cmd/client-gen", - "k8s.io/kubernetes/pkg/apis/componentconfig", + "k8s.io/kube-controller-manager/config/v1alpha1", "k8s.io/kubernetes/pkg/apis/core/v1/helper", "k8s.io/kubernetes/pkg/apis/storage/v1/util", "k8s.io/kubernetes/pkg/controller/endpoint", "k8s.io/kubernetes/pkg/kubelet/apis", - "k8s.io/kubernetes/pkg/util/goroutinemap", "k8s.io/kubernetes/pkg/util/mount", "k8s.io/kubernetes/pkg/util/version", "k8s.io/kubernetes/pkg/volume/flexvolume", diff --git a/Gopkg.toml b/Gopkg.toml index e094e3551..422eb6b46 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,5 +1,5 @@ -# Gopkg.toml +# Gopkg.toml # # Add constraints for packages that are a direct dependency # of this project (i.e. they are imported by golang source code) @@ -54,37 +54,45 @@ ignored = [ unused-packages = false # -# Kubernetes projects below should all be updated as a set +# Kubernetes projects below should all be updated as a set # as they are versioned together. # [[override]] name = "k8s.io/kubernetes" - version = "=v1.11.3" + version = "=v1.12.0" + +[[override]] + name = "k8s.io/kube-controller-manager" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/api" - version = "kubernetes-1.11.3" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/apiserver" - version = "kubernetes-1.11.3" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/apiextensions-apiserver" - version = "kubernetes-1.11.3" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/apimachinery" - version = "kubernetes-1.11.3" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/code-generator" - version = "kubernetes-1.11.3" + version = "kubernetes-1.12.0" [[override]] name = "k8s.io/client-go" - version = "v8.0.0" + version = "v9.0.0" + +[[override]] + name = "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner" + version = "v1.0.0" # kubernetes requires an older version of docker distribution [[override]] @@ -95,4 +103,4 @@ ignored = [ # and the Jenkins AMI doesn't have hg installed [[override]] name = "bitbucket.org/ww/goautoneg" - source = "github.com/rancher/goautoneg" \ No newline at end of file + source = "github.com/rancher/goautoneg" diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index 58a24d471..cb34a0798 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -66,6 +66,7 @@ rules: # PVs and PVCs are managed by the Rook provisioner - persistentvolumes - persistentvolumeclaims + - endpoints verbs: - get - list diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index f886d8bd1..d1007a2c0 100755 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -280,6 +280,7 @@ rules: # PVs and PVCs are managed by the Rook provisioner - persistentvolumes - persistentvolumeclaims + - endpoints verbs: - get - list diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 0dbbc39a9..323411682 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -31,15 +31,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) var parameterCodec = runtime.NewParameterCodec(scheme) - -func init() { - v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) - AddToScheme(scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + cassandrav1alpha1.AddToScheme, + cephv1beta1.AddToScheme, + cephv1.AddToScheme, + cockroachdbv1alpha1.AddToScheme, + edgefsv1alpha1.AddToScheme, + miniov1alpha1.AddToScheme, + nfsv1alpha1.AddToScheme, + rookv1alpha2.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition @@ -52,17 +58,13 @@ func init() { // ) // // kclientset, _ := kubernetes.NewForConfig(c) -// aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. -func AddToScheme(scheme *runtime.Scheme) { - cassandrav1alpha1.AddToScheme(scheme) - cephv1beta1.AddToScheme(scheme) - cephv1.AddToScheme(scheme) - cockroachdbv1alpha1.AddToScheme(scheme) - edgefsv1alpha1.AddToScheme(scheme) - miniov1alpha1.AddToScheme(scheme) - nfsv1alpha1.AddToScheme(scheme) - rookv1alpha2.AddToScheme(scheme) +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) } diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index b19875363..3b02ecd5c 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -31,15 +31,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) - -func init() { - v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - AddToScheme(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + cassandrav1alpha1.AddToScheme, + cephv1beta1.AddToScheme, + cephv1.AddToScheme, + cockroachdbv1alpha1.AddToScheme, + edgefsv1alpha1.AddToScheme, + miniov1alpha1.AddToScheme, + nfsv1alpha1.AddToScheme, + rookv1alpha2.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition @@ -52,17 +58,13 @@ func init() { // ) // // kclientset, _ := kubernetes.NewForConfig(c) -// aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. -func AddToScheme(scheme *runtime.Scheme) { - cassandrav1alpha1.AddToScheme(scheme) - cephv1beta1.AddToScheme(scheme) - cephv1.AddToScheme(scheme) - cockroachdbv1alpha1.AddToScheme(scheme) - edgefsv1alpha1.AddToScheme(scheme) - miniov1alpha1.AddToScheme(scheme) - nfsv1alpha1.AddToScheme(scheme) - rookv1alpha2.AddToScheme(scheme) +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) } diff --git a/pkg/operator/ceph/agent/types.go b/pkg/operator/ceph/agent/types.go index dd2f9dc0d..2d787e70b 100644 --- a/pkg/operator/ceph/agent/types.go +++ b/pkg/operator/ceph/agent/types.go @@ -19,7 +19,7 @@ package agent import ( "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/pkg/apis/componentconfig" + componentconfig "k8s.io/kube-controller-manager/config/v1alpha1" ) // Agent reference to be deployed diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index f6d53951a..b35d2ac3f 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -24,6 +24,7 @@ import ( "syscall" "github.com/coreos/pkg/capnslog" + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" opkit "github.com/rook/operator-kit" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume" @@ -35,7 +36,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/object/user" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/ceph/provisioner" - "github.com/rook/rook/pkg/operator/ceph/provisioner/controller" "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" diff --git a/pkg/operator/ceph/provisioner/controller/controller.go b/pkg/operator/ceph/provisioner/controller/controller.go deleted file mode 100644 index 7758b4501..000000000 --- a/pkg/operator/ceph/provisioner/controller/controller.go +++ /dev/null @@ -1,1107 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - "os/exec" - "reflect" - "strings" - "sync" - "time" - - "github.com/golang/glog" - "github.com/rook/rook/pkg/operator/ceph/provisioner/controller/leaderelection" - rl "github.com/rook/rook/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock" - "k8s.io/api/core/v1" - storage "k8s.io/api/storage/v1" - storagebeta "k8s.io/api/storage/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - corev1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" - ref "k8s.io/client-go/tools/reference" - "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/util/goroutinemap" - utilversion "k8s.io/kubernetes/pkg/util/version" -) - -// annClass annotation represents the storage class associated with a resource: -// - in PersistentVolumeClaim it represents required class to match. -// Only PersistentVolumes with the same class (i.e. annotation with the same -// value) can be bound to the claim. In case no such volume exists, the -// controller will provision a new one using StorageClass instance with -// the same name as the annotation value. -// - in PersistentVolume it represents storage class to which the persistent -// volume belongs. -const annClass = "volume.beta.kubernetes.io/storage-class" - -// This annotation is added to a PV that has been dynamically provisioned by -// Kubernetes. Its value is name of volume plugin that created the volume. -// It serves both user (to show where a PV comes from) and Kubernetes (to -// recognize dynamically provisioned PVs in its decisions). -const annDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" - -const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" - -// ProvisionController is a controller that provisions PersistentVolumes for -// PersistentVolumeClaims. -type ProvisionController struct { - client kubernetes.Interface - - // The name of the provisioner for which this controller dynamically - // provisions volumes. The value of annDynamicallyProvisioned and - // annStorageProvisioner to set & watch for, respectively - provisionerName string - - // The provisioner the controller will use to provision and delete volumes. - // Presumably this implementer of Provisioner carries its own - // volume-specific options and such that it needs in order to provision - // volumes. - provisioner Provisioner - - // Kubernetes cluster server version: - // * 1.4: storage classes introduced as beta. Technically out-of-tree dynamic - // provisioning is not officially supported, though it works - // * 1.5: storage classes stay in beta. Out-of-tree dynamic provisioning is - // officially supported - // * 1.6: storage classes enter GA - kubeVersion *utilversion.Version - - claimSource cache.ListerWatcher - claimController cache.Controller - volumeSource cache.ListerWatcher - volumeController cache.Controller - classSource cache.ListerWatcher - classReflector *cache.Reflector - - volumes cache.Store - claims cache.Store - classes cache.Store - - // Identity of this controller, generated at creation time and not persisted - // across restarts. Useful only for debugging, for seeing the source of - // events. controller.provisioner may have its own, different notion of - // identity which may/may not persist across restarts - identity types.UID - eventRecorder record.EventRecorder - - resyncPeriod time.Duration - - // Map of scheduled/running operations. - runningOperations goroutinemap.GoRoutineMap - - createProvisionedPVRetryCount int - createProvisionedPVInterval time.Duration - - failedProvisionThreshold, failedDeleteThreshold int - // Map of failed claims to provisions/volumes to deletes - failedProvisionStats, failedDeleteStats map[types.UID]int - failedProvisionStatsMutex, failedDeleteStatsMutex *sync.Mutex - - // Parameters of leaderelection.LeaderElectionConfig. Leader election is for - // when multiple controllers are running: they race to lock (lead) every PVC - // so that only one calls Provision for it (saving API calls, CPU cycles...) - leaseDuration, renewDeadline, retryPeriod, termLimit time.Duration - // Map of claim UID to LeaderElector: for checking if this controller - // is the leader of a given claim - leaderElectors map[types.UID]*leaderelection.LeaderElector - leaderElectorsMutex *sync.Mutex - - hasRun bool - hasRunLock *sync.Mutex -} - -const ( - // DefaultResyncPeriod is used when option function ResyncPeriod is omitted - DefaultResyncPeriod = 15 * time.Second - // DefaultExponentialBackOffOnError is used when option function ExponentialBackOffOnError is omitted - DefaultExponentialBackOffOnError = true - // DefaultCreateProvisionedPVRetryCount is used when option function CreateProvisionedPVRetryCount is omitted - DefaultCreateProvisionedPVRetryCount = 5 - // DefaultCreateProvisionedPVInterval is used when option function CreateProvisionedPVInterval is omitted - DefaultCreateProvisionedPVInterval = 10 * time.Second - // DefaultFailedProvisionThreshold is used when option function FailedProvisionThreshold is omitted - DefaultFailedProvisionThreshold = 15 - // DefaultFailedDeleteThreshold is used when option function FailedDeleteThreshold is omitted - DefaultFailedDeleteThreshold = 15 - // DefaultLeaseDuration is used when option function LeaseDuration is omitted - DefaultLeaseDuration = 15 * time.Second - // DefaultRenewDeadline is used when option function RenewDeadline is omitted - DefaultRenewDeadline = 10 * time.Second - // DefaultRetryPeriod is used when option function RetryPeriod is omitted - DefaultRetryPeriod = 2 * time.Second - // DefaultTermLimit is used when option function TermLimit is omitted - DefaultTermLimit = 30 * time.Second -) - -var errRuntime = fmt.Errorf("cannot call option functions after controller has Run") - -// ResyncPeriod is how often the controller relists PVCs, PVs, & storage -// classes. OnUpdate will be called even if nothing has changed, meaning failed -// operations may be retried on a PVC/PV every resyncPeriod regardless of -// whether it changed. Defaults to 15 seconds. -func ResyncPeriod(resyncPeriod time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.resyncPeriod = resyncPeriod - return nil - } -} - -// ExponentialBackOffOnError determines whether to exponentially back off from -// failures of Provision and Delete. Defaults to true. -func ExponentialBackOffOnError(exponentialBackOffOnError bool) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.runningOperations = goroutinemap.NewGoRoutineMap(exponentialBackOffOnError) - return nil - } -} - -// CreateProvisionedPVRetryCount is the number of retries when we create a PV -// object for a provisioned volume. Defaults to 5. -func CreateProvisionedPVRetryCount(createProvisionedPVRetryCount int) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.createProvisionedPVRetryCount = createProvisionedPVRetryCount - return nil - } -} - -// CreateProvisionedPVInterval is the interval between retries when we create a -// PV object for a provisioned volume. Defaults to 10 seconds. -func CreateProvisionedPVInterval(createProvisionedPVInterval time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.createProvisionedPVInterval = createProvisionedPVInterval - return nil - } -} - -// FailedProvisionThreshold is the threshold for max number of retries on -// failures of Provision. Defaults to 15. -func FailedProvisionThreshold(failedProvisionThreshold int) func(*ProvisionController) error { - return func(c *ProvisionController) error { - c.SetFailedProvisionThreshold(failedProvisionThreshold) - return nil - } -} - -// FailedDeleteThreshold is the threshold for max number of retries on failures -// of Delete. Defaults to 15. -func FailedDeleteThreshold(failedDeleteThreshold int) func(*ProvisionController) error { - return func(c *ProvisionController) error { - c.SetFailedDeleteThreshold(failedDeleteThreshold) - return nil - } -} - -// LeaseDuration is the duration that non-leader candidates will -// wait to force acquire leadership. This is measured against time of -// last observed ack. Defaults to 15 seconds. -func LeaseDuration(leaseDuration time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.leaseDuration = leaseDuration - return nil - } -} - -// RenewDeadline is the duration that the acting master will retry -// refreshing leadership before giving up. Defaults to 10 seconds. -func RenewDeadline(renewDeadline time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.renewDeadline = renewDeadline - return nil - } -} - -// RetryPeriod is the duration the LeaderElector clients should wait -// between tries of actions. Defaults to 2 seconds. -func RetryPeriod(retryPeriod time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.retryPeriod = retryPeriod - return nil - } -} - -// TermLimit is the maximum duration that a leader may remain the leader -// to complete the task before it must give up its leadership. 0 for forever -// or indefinite. Defaults to 30 seconds. -func TermLimit(termLimit time.Duration) func(*ProvisionController) error { - return func(c *ProvisionController) error { - if c.HasRun() { - return errRuntime - } - c.termLimit = termLimit - return nil - } -} - -// NewProvisionController creates a new provision controller -func NewProvisionController( - client kubernetes.Interface, - provisionerName string, - provisioner Provisioner, - kubeVersion string, - options ...func(*ProvisionController) error, -) *ProvisionController { - identity := uuid.NewUUID() - broadcaster := record.NewBroadcaster() - broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) - var eventRecorder record.EventRecorder - out, err := exec.Command("hostname").Output() - if err != nil { - eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s %s", provisionerName, string(identity))}) - } else { - eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s %s %s", provisionerName, strings.TrimSpace(string(out)), string(identity))}) - } - - // TODO: GetReference fails otherwise - v1.AddToScheme(scheme.Scheme) - - controller := &ProvisionController{ - client: client, - provisionerName: provisionerName, - provisioner: provisioner, - kubeVersion: utilversion.MustParseSemantic(kubeVersion), - identity: identity, - eventRecorder: eventRecorder, - resyncPeriod: DefaultResyncPeriod, - runningOperations: goroutinemap.NewGoRoutineMap(DefaultExponentialBackOffOnError), - createProvisionedPVRetryCount: DefaultCreateProvisionedPVRetryCount, - createProvisionedPVInterval: DefaultCreateProvisionedPVInterval, - failedProvisionThreshold: DefaultFailedProvisionThreshold, - failedDeleteThreshold: DefaultFailedDeleteThreshold, - failedProvisionStats: make(map[types.UID]int), - failedDeleteStats: make(map[types.UID]int), - failedProvisionStatsMutex: &sync.Mutex{}, - failedDeleteStatsMutex: &sync.Mutex{}, - leaseDuration: DefaultLeaseDuration, - renewDeadline: DefaultRenewDeadline, - retryPeriod: DefaultRetryPeriod, - termLimit: DefaultTermLimit, - leaderElectors: make(map[types.UID]*leaderelection.LeaderElector), - leaderElectorsMutex: &sync.Mutex{}, - hasRun: false, - hasRunLock: &sync.Mutex{}, - } - - for _, option := range options { - option(controller) - } - - controller.claimSource = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return client.CoreV1().PersistentVolumeClaims(v1.NamespaceAll).List(options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return client.CoreV1().PersistentVolumeClaims(v1.NamespaceAll).Watch(options) - }, - } - controller.claims, controller.claimController = cache.NewInformer( - controller.claimSource, - &v1.PersistentVolumeClaim{}, - controller.resyncPeriod, - cache.ResourceEventHandlerFuncs{ - AddFunc: controller.addClaim, - UpdateFunc: controller.updateClaim, - DeleteFunc: nil, - }, - ) - - controller.volumeSource = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return client.CoreV1().PersistentVolumes().List(options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return client.CoreV1().PersistentVolumes().Watch(options) - }, - } - controller.volumes, controller.volumeController = cache.NewInformer( - controller.volumeSource, - &v1.PersistentVolume{}, - controller.resyncPeriod, - cache.ResourceEventHandlerFuncs{ - AddFunc: nil, - UpdateFunc: controller.updateVolume, - DeleteFunc: nil, - }, - ) - - controller.classes = cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) - if controller.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) { - controller.classSource = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return client.StorageV1().StorageClasses().List(options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return client.StorageV1().StorageClasses().Watch(options) - }, - } - controller.classReflector = cache.NewReflector( - controller.classSource, - &storage.StorageClass{}, - controller.classes, - controller.resyncPeriod, - ) - } else { - controller.classSource = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return client.StorageV1beta1().StorageClasses().List(options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return client.StorageV1beta1().StorageClasses().Watch(options) - }, - } - controller.classReflector = cache.NewReflector( - controller.classSource, - &storagebeta.StorageClass{}, - controller.classes, - controller.resyncPeriod, - ) - } - - return controller -} - -// Run starts all of this controller's control loops -func (ctrl *ProvisionController) Run(stopCh <-chan struct{}) { - glog.Infof("Starting provisioner controller %s!", string(ctrl.identity)) - ctrl.hasRunLock.Lock() - ctrl.hasRun = true - ctrl.hasRunLock.Unlock() - go ctrl.claimController.Run(stopCh) - go ctrl.volumeController.Run(stopCh) - go ctrl.classReflector.Run(stopCh) - <-stopCh -} - -// HasRun returns whether the controller has Run -func (ctrl *ProvisionController) HasRun() bool { - ctrl.hasRunLock.Lock() - defer ctrl.hasRunLock.Unlock() - return ctrl.hasRun -} - -// SetFailedProvisionThreshold sets the value of failedProvisionThreshold -func (ctrl *ProvisionController) SetFailedProvisionThreshold(threshold int) { - ctrl.failedProvisionStatsMutex.Lock() - ctrl.failedProvisionThreshold = threshold - ctrl.failedProvisionStatsMutex.Unlock() -} - -// SetFailedDeleteThreshold sets the value of failedDeleteThreshold -func (ctrl *ProvisionController) SetFailedDeleteThreshold(threshold int) { - ctrl.failedDeleteStatsMutex.Lock() - ctrl.failedDeleteThreshold = threshold - ctrl.failedDeleteStatsMutex.Unlock() -} - -// On add claim, check if the added claim should have a volume provisioned for -// it and provision one if so. -func (ctrl *ProvisionController) addClaim(obj interface{}) { - claim, ok := obj.(*v1.PersistentVolumeClaim) - if !ok { - glog.Errorf("Expected PersistentVolumeClaim but addClaim received %+v", obj) - return - } - - if ctrl.shouldProvision(claim) { - ctrl.leaderElectorsMutex.Lock() - le, ok := ctrl.leaderElectors[claim.UID] - ctrl.leaderElectorsMutex.Unlock() - if ok && le.IsLeader() { - opName := fmt.Sprintf("provision-%s[%s]", claimToClaimKey(claim), string(claim.UID)) - ctrl.scheduleOperation(opName, func() error { - err := ctrl.provisionClaimOperation(claim) - ctrl.updateProvisionStats(claim, err) - return err - }) - } else { - opName := fmt.Sprintf("lock-provision-%s[%s]", claimToClaimKey(claim), string(claim.UID)) - ctrl.scheduleOperation(opName, func() error { - ctrl.lockProvisionClaimOperation(claim) - return nil - }) - } - } -} - -// On update claim, pass the new claim to addClaim. Updates occur at least every -// resyncPeriod. -func (ctrl *ProvisionController) updateClaim(oldObj, newObj interface{}) { - // If they are exactly the same it must be a forced resync (every - // resyncPeriod). - if reflect.DeepEqual(oldObj, newObj) { - ctrl.addClaim(newObj) - return - } - - // If not a forced resync, we filter out the frequent leader election record - // annotation changes by checking if the only update is in the annotation - oldClaim, ok := oldObj.(*v1.PersistentVolumeClaim) - if !ok { - glog.Errorf("Expected PersistentVolumeClaim but handler received %#v", oldObj) - return - } - newClaim, ok := newObj.(*v1.PersistentVolumeClaim) - if !ok { - glog.Errorf("Expected PersistentVolumeClaim but handler received %#v", newObj) - return - } - - skipAddClaim, err := ctrl.isOnlyRecordUpdate(oldClaim, newClaim) - if err != nil { - glog.Errorf("Error checking if only record was updated in claim: %v", oldClaim) - return - } - - if !skipAddClaim { - ctrl.addClaim(newObj) - } -} - -// On update volume, check if the updated volume should be deleted and delete if -// so. Updates occur at least every resyncPeriod. -func (ctrl *ProvisionController) updateVolume(oldObj, newObj interface{}) { - volume, ok := newObj.(*v1.PersistentVolume) - if !ok { - glog.Errorf("Expected PersistentVolume but handler received %#v", newObj) - return - } - - if ctrl.shouldDelete(volume) { - opName := fmt.Sprintf("delete-%s[%s]", volume.Name, string(volume.UID)) - ctrl.scheduleOperation(opName, func() error { - err := ctrl.deleteVolumeOperation(volume) - ctrl.updateDeleteStats(volume, err) - return err - }) - } -} - -// isOnlyRecordUpdate checks if the only update between the old & new claim is -// the leader election record annotation. -func (ctrl *ProvisionController) isOnlyRecordUpdate(oldClaim, newClaim *v1.PersistentVolumeClaim) (bool, error) { - old, err := ctrl.removeRecord(oldClaim) - if err != nil { - return false, err - } - new, err := ctrl.removeRecord(newClaim) - if err != nil { - return false, err - } - return reflect.DeepEqual(old, new), nil -} - -// removeRecord returns a claim with its leader election record annotation and -// ResourceVersion set blank -func (ctrl *ProvisionController) removeRecord(claim *v1.PersistentVolumeClaim) (*v1.PersistentVolumeClaim, error) { - claimClone := claim.DeepCopy() - - if claimClone.Annotations == nil { - claimClone.Annotations = make(map[string]string) - } - claimClone.Annotations[rl.LeaderElectionRecordAnnotationKey] = "" - - claimClone.ResourceVersion = "" - - return claimClone, nil -} - -func (ctrl *ProvisionController) shouldProvision(claim *v1.PersistentVolumeClaim) bool { - ctrl.failedProvisionStatsMutex.Lock() - if failureCount, exists := ctrl.failedProvisionStats[claim.UID]; exists == true { - if failureCount >= ctrl.failedProvisionThreshold && ctrl.failedProvisionThreshold > 0 { - glog.Errorf("Exceeded failedProvisionThreshold threshold: %d, for claim %q, provisioner will not attempt retries for this claim", ctrl.failedProvisionThreshold, claimToClaimKey(claim)) - ctrl.failedProvisionStatsMutex.Unlock() - return false - } - } - ctrl.failedProvisionStatsMutex.Unlock() - - if claim.Spec.VolumeName != "" { - return false - } - - // Kubernetes 1.5 provisioning with annStorageProvisioner - if provisioner, found := claim.Annotations[annStorageProvisioner]; found { - if provisioner == ctrl.provisionerName { - return true - } - return false - } - - // Kubernetes 1.4 provisioning, evaluating class.Provisioner - claimClass := helper.GetPersistentVolumeClaimClass(claim) - provisioner, _, _, err := ctrl.getStorageClassFields(claimClass) - if err != nil { - glog.Errorf("Error getting claim %q's StorageClass's fields: %v", claimToClaimKey(claim), err) - return false - } - if provisioner != ctrl.provisionerName { - return false - } - - return true -} - -func (ctrl *ProvisionController) shouldDelete(volume *v1.PersistentVolume) bool { - ctrl.failedDeleteStatsMutex.Lock() - if failureCount, exists := ctrl.failedDeleteStats[volume.UID]; exists == true { - if failureCount >= ctrl.failedDeleteThreshold && ctrl.failedDeleteThreshold > 0 { - glog.Errorf("Exceeded failedDeleteThreshold threshold: %d, for volume %q, provisioner will not attempt retries for this volume", ctrl.failedDeleteThreshold, volume.Name) - ctrl.failedDeleteStatsMutex.Unlock() - return false - } - } - ctrl.failedDeleteStatsMutex.Unlock() - - // In 1.5+ we delete only if the volume is in state Released. In 1.4 we must - // delete if the volume is in state Failed too. - if ctrl.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.5.0")) { - if volume.Status.Phase != v1.VolumeReleased { - return false - } - } else { - if volume.Status.Phase != v1.VolumeReleased && volume.Status.Phase != v1.VolumeFailed { - return false - } - } - - if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimDelete { - return false - } - - if !metav1.HasAnnotation(volume.ObjectMeta, annDynamicallyProvisioned) { - return false - } - - if ann := volume.Annotations[annDynamicallyProvisioned]; ann != ctrl.provisionerName { - return false - } - - return true -} - -// lockProvisionClaimOperation wraps provisionClaimOperation. In case other -// controllers are serving the same claims, to prevent them all from creating -// volumes for a claim & racing to submit their PV, each controller creates a -// LeaderElector to instead race for the leadership (lock), where only the -// leader is tasked with provisioning & may try to do so -func (ctrl *ProvisionController) lockProvisionClaimOperation(claim *v1.PersistentVolumeClaim) { - stoppedLeading := false - rl := rl.ProvisionPVCLock{ - PVCMeta: claim.ObjectMeta, - Client: ctrl.client, - LockConfig: rl.Config{ - Identity: string(ctrl.identity), - EventRecorder: ctrl.eventRecorder, - }, - } - le, err := leaderelection.NewLeaderElector(leaderelection.Config{ - Lock: &rl, - LeaseDuration: ctrl.leaseDuration, - RenewDeadline: ctrl.renewDeadline, - RetryPeriod: ctrl.retryPeriod, - TermLimit: ctrl.termLimit, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: func(_ <-chan struct{}) { - opName := fmt.Sprintf("provision-%s[%s]", claimToClaimKey(claim), string(claim.UID)) - ctrl.scheduleOperation(opName, func() error { - err := ctrl.provisionClaimOperation(claim) - ctrl.updateProvisionStats(claim, err) - return err - }) - }, - OnStoppedLeading: func() { - stoppedLeading = true - }, - }, - }) - if err != nil { - glog.Errorf("Error creating LeaderElector, can't provision for claim %q: %v", claimToClaimKey(claim), err) - return - } - - ctrl.leaderElectorsMutex.Lock() - ctrl.leaderElectors[claim.UID] = le - ctrl.leaderElectorsMutex.Unlock() - - // To determine when to stop trying to acquire/renew the lock, watch for - // provisioning success/failure. (The leader could get the result of its - // operation but it has to watch anyway) - stopCh := make(chan struct{}) - successCh, err := ctrl.watchProvisioning(claim, stopCh) - if err != nil { - glog.Errorf("Error watching for provisioning success, can't provision for claim %q: %v", claimToClaimKey(claim), err) - } - - le.Run(successCh) - - close(stopCh) - - // If we were the leader and stopped, give others a chance to acquire - // (whether they exist & want to or not). Else, there must have been a - // success so just proceed. - if stoppedLeading { - time.Sleep(ctrl.leaseDuration + ctrl.retryPeriod) - } - - ctrl.leaderElectorsMutex.Lock() - delete(ctrl.leaderElectors, claim.UID) - ctrl.leaderElectorsMutex.Unlock() -} - -func (ctrl *ProvisionController) updateProvisionStats(claim *v1.PersistentVolumeClaim, err error) { - ctrl.failedProvisionStatsMutex.Lock() - defer ctrl.failedProvisionStatsMutex.Unlock() - - // Do not record the failed claim info when failedProvisionThreshold is not set - if ctrl.failedProvisionThreshold <= 0 { - return - } - - if err != nil { - if failureCount, exists := ctrl.failedProvisionStats[claim.UID]; exists == true { - failureCount = failureCount + 1 - ctrl.failedProvisionStats[claim.UID] = failureCount - } else { - ctrl.failedProvisionStats[claim.UID] = 1 - } - } else { - delete(ctrl.failedProvisionStats, claim.UID) - } -} - -func (ctrl *ProvisionController) updateDeleteStats(volume *v1.PersistentVolume, err error) { - ctrl.failedDeleteStatsMutex.Lock() - defer ctrl.failedDeleteStatsMutex.Unlock() - - // Do not record the failed volume info when failedDeleteThreshold is not set - if ctrl.failedDeleteThreshold <= 0 { - return - } - - if err != nil { - if failureCount, exists := ctrl.failedDeleteStats[volume.UID]; exists == true { - failureCount = failureCount + 1 - ctrl.failedDeleteStats[volume.UID] = failureCount - } else { - ctrl.failedDeleteStats[volume.UID] = 1 - } - } else { - delete(ctrl.failedDeleteStats, volume.UID) - } -} - -// provisionClaimOperation attempts to provision a volume for the given claim. -// Returns an error for use by goroutinemap when expbackoff is enabled: if nil, -// the operation is deleted, else the operation may be retried with expbackoff. -func (ctrl *ProvisionController) provisionClaimOperation(claim *v1.PersistentVolumeClaim) error { - // Most code here is identical to that found in controller.go of kube's PV controller... - claimClass := helper.GetPersistentVolumeClaimClass(claim) - glog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) - - // A previous doProvisionClaim may just have finished while we were waiting for - // the locks. Check that PV (with deterministic name) hasn't been provisioned - // yet. - pvName := ctrl.getProvisionedVolumeNameForClaim(claim) - volume, err := ctrl.client.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) - if err == nil && volume != nil { - // Volume has been already provisioned, nothing to do. - glog.V(4).Infof("provisionClaimOperation [%s]: volume already exists, skipping", claimToClaimKey(claim)) - return nil - } - - // Prepare a claimRef to the claim early (to fail before a volume is - // provisioned) - claimRef, err := ref.GetReference(scheme.Scheme, claim) - if err != nil { - glog.Errorf("Unexpected error getting claim reference to claim %q: %v", claimToClaimKey(claim), err) - return nil - } - - provisioner, reclaimPolicyField, parameters, err := ctrl.getStorageClassFields(claimClass) - if err != nil { - glog.Errorf("Error getting claim %q's StorageClass's fields: %v", claimToClaimKey(claim), err) - return nil - } - if provisioner != ctrl.provisionerName { - // class.Provisioner has either changed since shouldProvision() or - // annDynamicallyProvisioned contains different provisioner than - // class.Provisioner. - glog.Errorf("Unknown provisioner %q requested in claim %q's StorageClass %q", provisioner, claimToClaimKey(claim), claimClass) - return nil - } - - var reclaimPolicy v1.PersistentVolumeReclaimPolicy - if reclaimPolicyField == nil { - reclaimPolicy = v1.PersistentVolumeReclaimDelete - } else { - reclaimPolicy = *reclaimPolicyField - } - - options := VolumeOptions{ - PersistentVolumeReclaimPolicy: reclaimPolicy, - PVName: pvName, - PVC: claim, - Parameters: parameters, - } - - ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, "Provisioning", fmt.Sprintf("External provisioner is provisioning volume for claim %q", claimToClaimKey(claim))) - - volume, err = ctrl.provisioner.Provision(options) - if err != nil { - if ierr, ok := err.(*IgnoredError); ok { - // Provision ignored, do nothing and hope another provisioner will provision it. - glog.Infof("provision of claim %q ignored: %v", claimToClaimKey(claim), ierr) - return nil - } - strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", claimClass, err) - glog.Errorf("Failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), claimClass, err) - ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", strerr) - return err - } - - glog.Infof("volume %q for claim %q created", volume.Name, claimToClaimKey(claim)) - - // Set ClaimRef and the PV controller will bind and set annBoundByController for us - volume.Spec.ClaimRef = claimRef - - metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annDynamicallyProvisioned, ctrl.provisionerName) - if ctrl.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) { - volume.Spec.StorageClassName = claimClass - } else { - metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annClass, claimClass) - } - - // Try to create the PV object several times - for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ { - glog.V(4).Infof("provisionClaimOperation [%s]: trying to save volume %s", claimToClaimKey(claim), volume.Name) - if _, err = ctrl.client.CoreV1().PersistentVolumes().Create(volume); err == nil { - // Save succeeded. - glog.Infof("volume %q for claim %q saved", volume.Name, claimToClaimKey(claim)) - break - } - // Save failed, try again after a while. - glog.Infof("failed to save volume %q for claim %q: %v", volume.Name, claimToClaimKey(claim), err) - time.Sleep(ctrl.createProvisionedPVInterval) - } - - if err != nil { - // Save failed. Now we have a storage asset outside of Kubernetes, - // but we don't have appropriate PV object for it. - // Emit some event here and try to delete the storage asset several - // times. - strerr := fmt.Sprintf("Error creating provisioned PV object for claim %s: %v. Deleting the volume.", claimToClaimKey(claim), err) - glog.Error(strerr) - ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningFailed", strerr) - - for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ { - if err = ctrl.provisioner.Delete(volume); err == nil { - // Delete succeeded - glog.V(4).Infof("provisionClaimOperation [%s]: cleaning volume %s succeeded", claimToClaimKey(claim), volume.Name) - break - } - // Delete failed, try again after a while. - glog.Infof("failed to delete volume %q: %v", volume.Name, err) - time.Sleep(ctrl.createProvisionedPVInterval) - } - - if err != nil { - // Delete failed several times. There is an orphaned volume and there - // is nothing we can do about it. - strerr := fmt.Sprintf("Error cleaning provisioned volume for claim %s: %v. Please delete manually.", claimToClaimKey(claim), err) - glog.Error(strerr) - ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "ProvisioningCleanupFailed", strerr) - } - } else { - glog.Infof("volume %q provisioned for claim %q", volume.Name, claimToClaimKey(claim)) - msg := fmt.Sprintf("Successfully provisioned volume %s", volume.Name) - ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, "ProvisioningSucceeded", msg) - } - - return nil -} - -// watchProvisioning returns a channel to which it sends the results of all -// provisioning attempts for the given claim. The PVC being modified to no -// longer need provisioning is considered a success. -func (ctrl *ProvisionController) watchProvisioning(claim *v1.PersistentVolumeClaim, stopChannel chan struct{}) (<-chan bool, error) { - stopWatchPVC := make(chan struct{}) - pvcCh, err := ctrl.watchPVC(claim, stopWatchPVC) - if err != nil { - glog.Infof("cannot start watcher for PVC %s/%s: %v", claim.Namespace, claim.Name, err) - return nil, err - } - - successCh := make(chan bool, 0) - - go func() { - defer close(stopWatchPVC) - defer close(successCh) - - for { - select { - case _ = <-stopChannel: - return - - case event := <-pvcCh: - switch event.Object.(type) { - case *v1.PersistentVolumeClaim: - // PVC changed - claim := event.Object.(*v1.PersistentVolumeClaim) - glog.V(4).Infof("claim update received: %s %s/%s %s", event.Type, claim.Namespace, claim.Name, claim.Status.Phase) - switch event.Type { - case watch.Added, watch.Modified: - if claim.Spec.VolumeName != "" { - successCh <- true - } else if !ctrl.shouldProvision(claim) { - glog.Infof("claim %s/%s was modified to not ask for this provisioner", claim.Namespace, claim.Name) - successCh <- true - } - - case watch.Deleted: - glog.Infof("claim %s/%s was deleted", claim.Namespace, claim.Name) - successCh <- true - - case watch.Error: - glog.Infof("claim %s/%s watcher failed", claim.Namespace, claim.Name) - successCh <- true - default: - } - case *v1.Event: - // Event received - claimEvent := event.Object.(*v1.Event) - glog.V(4).Infof("claim event received: %s %s/%s %s/%s %s", event.Type, claimEvent.Namespace, claimEvent.Name, claimEvent.InvolvedObject.Namespace, claimEvent.InvolvedObject.Name, claimEvent.Reason) - if claimEvent.Reason == "ProvisioningSucceeded" { - successCh <- true - } else if claimEvent.Reason == "ProvisioningFailed" { - successCh <- false - } - } - } - } - }() - - return successCh, nil -} - -// watchPVC returns a watch on the given PVC and ProvisioningFailed & -// ProvisioningSucceeded events involving it -func (ctrl *ProvisionController) watchPVC(claim *v1.PersistentVolumeClaim, stopChannel chan struct{}) (<-chan watch.Event, error) { - options := metav1.ListOptions{ - FieldSelector: "metadata.name=" + claim.Name, - Watch: true, - ResourceVersion: claim.ResourceVersion, - } - - pvcWatch, err := ctrl.claimSource.Watch(options) - if err != nil { - return nil, err - } - - failWatch, err := ctrl.getPVCEventWatch(claim, v1.EventTypeWarning, "ProvisioningFailed") - if err != nil { - pvcWatch.Stop() - return nil, err - } - - successWatch, err := ctrl.getPVCEventWatch(claim, v1.EventTypeNormal, "ProvisioningSucceeded") - if err != nil { - failWatch.Stop() - pvcWatch.Stop() - return nil, err - } - - eventCh := make(chan watch.Event, 0) - - go func() { - defer successWatch.Stop() - defer failWatch.Stop() - defer pvcWatch.Stop() - defer close(eventCh) - - for { - select { - case _ = <-stopChannel: - return - - case pvcEvent, ok := <-pvcWatch.ResultChan(): - if !ok { - return - } - eventCh <- pvcEvent - - case failEvent, ok := <-failWatch.ResultChan(): - if !ok { - return - } - eventCh <- failEvent - - case successEvent, ok := <-successWatch.ResultChan(): - if !ok { - return - } - eventCh <- successEvent - } - } - }() - - return eventCh, nil -} - -// getPVCEventWatch returns a watch on the given PVC for the given event from -// this point forward. -func (ctrl *ProvisionController) getPVCEventWatch(claim *v1.PersistentVolumeClaim, eventType, reason string) (watch.Interface, error) { - claimKind := "PersistentVolumeClaim" - claimUID := string(claim.UID) - fieldSelector := ctrl.client.CoreV1().Events(claim.Namespace).GetFieldSelector(&claim.Name, &claim.Namespace, &claimKind, &claimUID).String() + ",type=" + eventType + ",reason=" + reason - - list, err := ctrl.client.CoreV1().Events(claim.Namespace).List(metav1.ListOptions{ - FieldSelector: fieldSelector, - }) - if err != nil { - return nil, err - } - - resourceVersion := "" - if len(list.Items) >= 1 { - resourceVersion = list.Items[len(list.Items)-1].ResourceVersion - } - - return ctrl.client.CoreV1().Events(claim.Namespace).Watch(metav1.ListOptions{ - FieldSelector: fieldSelector, - Watch: true, - ResourceVersion: resourceVersion, - }) -} - -func (ctrl *ProvisionController) deleteVolumeOperation(volume *v1.PersistentVolume) error { - glog.V(4).Infof("deleteVolumeOperation [%s] started", volume.Name) - - // This method may have been waiting for a volume lock for some time. - // Our check does not have to be as sophisticated as PV controller's, we can - // trust that the PV controller has set the PV to Released/Failed and it's - // ours to delete - newVolume, err := ctrl.client.CoreV1().PersistentVolumes().Get(volume.Name, metav1.GetOptions{}) - if err != nil { - return nil - } - if !ctrl.shouldDelete(newVolume) { - glog.Infof("volume %q no longer needs deletion, skipping", volume.Name) - return nil - } - - err = ctrl.provisioner.Delete(volume) - if err != nil { - if ierr, ok := err.(*IgnoredError); ok { - // Delete ignored, do nothing and hope another provisioner will delete it. - glog.Infof("deletion of volume %q ignored: %v", volume.Name, ierr) - return nil - } - // Delete failed, emit an event. - glog.Errorf("Deletion of volume %q failed: %v", volume.Name, err) - ctrl.eventRecorder.Event(volume, v1.EventTypeWarning, "VolumeFailedDelete", err.Error()) - return err - } - - glog.Infof("volume %q deleted", volume.Name) - - glog.V(4).Infof("deleteVolumeOperation [%s]: success", volume.Name) - // Delete the volume - if err = ctrl.client.CoreV1().PersistentVolumes().Delete(volume.Name, nil); err != nil { - // Oops, could not delete the volume and therefore the controller will - // try to delete the volume again on next update. - glog.Infof("failed to delete volume %q from database: %v", volume.Name, err) - return nil - } - - glog.Infof("volume %q deleted from database", volume.Name) - return nil -} - -// getProvisionedVolumeNameForClaim returns PV.Name for the provisioned volume. -// The name must be unique. -func (ctrl *ProvisionController) getProvisionedVolumeNameForClaim(claim *v1.PersistentVolumeClaim) string { - return "pvc-" + string(claim.UID) -} - -// scheduleOperation starts given asynchronous operation on given volume. It -// makes sure the operation is already not running. -func (ctrl *ProvisionController) scheduleOperation(operationName string, operation func() error) { - glog.Infof("scheduleOperation[%s]", operationName) - - err := ctrl.runningOperations.Run(operationName, operation) - if err != nil { - if goroutinemap.IsAlreadyExists(err) { - glog.V(4).Infof("operation %q is already running, skipping", operationName) - } else { - glog.Errorf("Error scheduling operation %q: %v", operationName, err) - } - } -} - -func (ctrl *ProvisionController) getStorageClassFields(name string) (string, *v1.PersistentVolumeReclaimPolicy, map[string]string, error) { - classObj, found, err := ctrl.classes.GetByKey(name) - if err != nil { - return "", nil, nil, err - } - if !found { - return "", nil, nil, fmt.Errorf("StorageClass %q not found", name) - // 3. It tries to find a StorageClass instance referenced by annotation - // `claim.Annotations["volume.beta.kubernetes.io/storage-class"]`. If not - // found, it SHOULD report an error (by sending an event to the claim) and it - // SHOULD retry periodically with step i. - } - switch class := classObj.(type) { - case *storage.StorageClass: - return class.Provisioner, class.ReclaimPolicy, class.Parameters, nil - case *storagebeta.StorageClass: - return class.Provisioner, class.ReclaimPolicy, class.Parameters, nil - } - return "", nil, nil, fmt.Errorf("Cannot convert object to StorageClass: %+v", classObj) -} - -func claimToClaimKey(claim *v1.PersistentVolumeClaim) string { - return fmt.Sprintf("%s/%s", claim.Namespace, claim.Name) -} diff --git a/pkg/operator/ceph/provisioner/controller/controller_test.go b/pkg/operator/ceph/provisioner/controller/controller_test.go deleted file mode 100644 index 39397b698..000000000 --- a/pkg/operator/ceph/provisioner/controller/controller_test.go +++ /dev/null @@ -1,711 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "errors" - "fmt" - "reflect" - "strconv" - "sync" - "testing" - "time" - - rl "github.com/rook/rook/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock" - "k8s.io/api/core/v1" - storagebeta "k8s.io/api/storage/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/kubernetes/scheme" - fakev1core "k8s.io/client-go/kubernetes/typed/core/v1/fake" - testclient "k8s.io/client-go/testing" - fcache "k8s.io/client-go/tools/cache/testing" - ref "k8s.io/client-go/tools/reference" -) - -const ( - resyncPeriod = 100 * time.Millisecond -) - -// TODO clean this up, e.g. remove redundant params (provisionerName: "foo.bar/baz") -func TestController(t *testing.T) { - tests := []struct { - name string - objs []runtime.Object - provisionerName string - provisioner Provisioner - verbs []string - reaction testclient.ReactionFunc - expectedVolumes []v1.PersistentVolume - }{ - { - name: "provision for claim-1 but not claim-2", - objs: []runtime.Object{ - newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - newStorageClass("class-2", "abc.def/ghi", v1.PersistentVolumeReclaimDelete), - newClaim("claim-1", "uid-1-1", "class-1", "", nil), - newClaim("claim-2", "uid-1-2", "class-2", "", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newProvisionedVolume(newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), newClaim("claim-1", "uid-1-1", "class-1", "", nil)), - }, - }, - { - name: "delete volume-1 but not volume-2", - objs: []runtime.Object{ - newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - newVolume("volume-2", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "abc.def/ghi"}), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newVolume("volume-2", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "abc.def/ghi"}), - }, - }, - { - name: "don't provision for claim-1 because it's already bound", - objs: []runtime.Object{ - newClaim("claim-1", "uid-1-1", "class-1", "volume-1", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume(nil), - }, - { - name: "don't provision for claim-1 because its class doesn't exist", - objs: []runtime.Object{ - newClaim("claim-1", "uid-1-1", "class-1", "", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume(nil), - }, - { - name: "don't delete volume-1 because it's still bound", - objs: []runtime.Object{ - newVolume("volume-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newVolume("volume-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - }, - { - name: "don't delete volume-1 because its reclaim policy is not delete", - objs: []runtime.Object{ - newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - }, - { - name: "provisioner fails to provision for claim-1: no pv is created", - objs: []runtime.Object{ - newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - newClaim("claim-1", "uid-1-1", "class-1", "", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newBadTestProvisioner(), - expectedVolumes: []v1.PersistentVolume(nil), - }, - { - name: "provisioner fails to delete volume-1: pv is not deleted", - objs: []runtime.Object{ - newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - provisionerName: "foo.bar/baz", - provisioner: newBadTestProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - }, - { - name: "try to provision for claim-1 but fail to save the pv object", - objs: []runtime.Object{ - newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - newClaim("claim-1", "uid-1-1", "class-1", "", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - verbs: []string{"create"}, - reaction: func(action testclient.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, errors.New("fake error") - }, - expectedVolumes: []v1.PersistentVolume(nil), - }, - { - name: "try to delete volume-1 but fail to delete the pv object", - objs: []runtime.Object{ - newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - provisionerName: "foo.bar/baz", - provisioner: newTestProvisioner(), - verbs: []string{"delete"}, - reaction: func(action testclient.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, errors.New("fake error") - }, - expectedVolumes: []v1.PersistentVolume{ - *newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - }, - }, - { - name: "provision for claim-1 but not claim-2, because it is ignored", - objs: []runtime.Object{ - newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - newClaim("claim-1", "uid-1-1", "class-1", "", nil), - newClaim("claim-2", "uid-1-2", "class-1", "", nil), - }, - provisionerName: "foo.bar/baz", - provisioner: newIgnoredProvisioner(), - expectedVolumes: []v1.PersistentVolume{ - *newProvisionedVolume(newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), newClaim("claim-1", "uid-1-1", "class-1", "", nil)), - }, - }, - } - for _, test := range tests { - client := fake.NewSimpleClientset(test.objs...) - if len(test.verbs) != 0 { - for _, v := range test.verbs { - client.Fake.PrependReactor(v, "persistentvolumes", test.reaction) - } - } - ctrl := newTestProvisionController(client, test.provisionerName, test.provisioner, "v1.5.0") - stopCh := make(chan struct{}) - go ctrl.Run(stopCh) - - time.Sleep(2 * resyncPeriod) - ctrl.runningOperations.Wait() - - pvList, _ := client.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) - if !reflect.DeepEqual(test.expectedVolumes, pvList.Items) { - t.Logf("test case: %s", test.name) - t.Errorf("expected PVs:\n %v\n but got:\n %v\n", test.expectedVolumes, pvList.Items) - } - close(stopCh) - } -} - -func TestMultipleControllers(t *testing.T) { - tests := []struct { - name string - provisionerName string - numControllers int - numClaims int - expectedCalls int - }{ - { - name: "call provision exactly once", - provisionerName: "foo.bar/baz", - numControllers: 5, - numClaims: 1, - expectedCalls: 1, - }, - } - for _, test := range tests { - client := fake.NewSimpleClientset() - - // Create a reactor to reject Updates if object has already been modified, - // like etcd. - claimSource := fcache.NewFakePVCControllerSource() - reactor := claimReactor{ - fake: &fakev1core.FakeCoreV1{Fake: &client.Fake}, - claims: make(map[string]*v1.PersistentVolumeClaim), - lock: sync.Mutex{}, - claimSource: claimSource, - } - reactor.claims["claim-1"] = newClaim("claim-1", "uid-1-1", "class-1", "", nil) - client.PrependReactor("update", "persistentvolumeclaims", reactor.React) - client.PrependReactor("get", "persistentvolumeclaims", reactor.React) - - // Create a fake watch so each controller can get ProvisioningSucceeded - fakeWatch := watch.NewFakeWithChanSize(test.numControllers, false) - client.PrependWatchReactor("events", testclient.DefaultWatchReactor(fakeWatch, nil)) - client.PrependReactor("create", "events", func(action testclient.Action) (bool, runtime.Object, error) { - obj := action.(testclient.CreateAction).GetObject() - for i := 0; i < test.numControllers; i++ { - fakeWatch.Add(obj) - } - return true, obj, nil - }) - - provisioner := newTestProvisioner() - ctrls := make([]*ProvisionController, test.numControllers) - stopChs := make([]chan struct{}, test.numControllers) - for i := 0; i < test.numControllers; i++ { - ctrls[i] = NewProvisionController(client, test.provisionerName, provisioner, "v1.5.0", CreateProvisionedPVInterval(10*time.Millisecond)) - ctrls[i].claimSource = claimSource - ctrls[i].claims.Add(newClaim("claim-1", "uid-1-1", "class-1", "", nil)) - ctrls[i].classes.Add(newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete)) - stopChs[i] = make(chan struct{}) - } - - for i := 0; i < test.numControllers; i++ { - go ctrls[i].addClaim(newClaim("claim-1", "uid-1-1", "class-1", "", nil)) - } - - // Sleep for 3 election retry periods - time.Sleep(3 * ctrls[0].retryPeriod) - - if test.expectedCalls != len(provisioner.provisionCalls) { - t.Logf("test case: %s", test.name) - t.Errorf("expected provision calls:\n %v\n but got:\n %v\n", test.expectedCalls, len(provisioner.provisionCalls)) - } - - for _, stopCh := range stopChs { - close(stopCh) - } - } -} - -func TestShouldProvision(t *testing.T) { - tests := []struct { - name string - provisionerName string - class *storagebeta.StorageClass - claim *v1.PersistentVolumeClaim - expectedShould bool - }{ - { - name: "should provision", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-1", "", nil), - expectedShould: true, - }, - { - name: "claim already bound", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-1", "foo", nil), - expectedShould: false, - }, - { - name: "no such class", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-2", "", nil), - expectedShould: false, - }, - { - name: "not this provisioner's job", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-1", "abc.def/ghi", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-1", "", nil), - expectedShould: false, - }, - // Kubernetes 1.5 provisioning - annStorageProvisioner is set - // and only this annotation is evaluated - { - name: "should provision 1.5", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-2", "abc.def/ghi", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-1", "", - map[string]string{annStorageProvisioner: "foo.bar/baz"}), - expectedShould: true, - }, - { - name: "unknown provisioner 1.5", - provisionerName: "foo.bar/baz", - class: newStorageClass("class-1", "foo.bar/baz", v1.PersistentVolumeReclaimDelete), - claim: newClaim("claim-1", "1-1", "class-1", "", - map[string]string{annStorageProvisioner: "abc.def/ghi"}), - expectedShould: false, - }, - } - for _, test := range tests { - client := fake.NewSimpleClientset(test.claim) - provisioner := newTestProvisioner() - ctrl := newTestProvisionController(client, test.provisionerName, provisioner, "v1.5.0") - - err := ctrl.classes.Add(test.class) - if err != nil { - t.Logf("test case: %s", test.name) - t.Errorf("error adding class %v to cache: %v", test.class, err) - } - - should := ctrl.shouldProvision(test.claim) - if test.expectedShould != should { - t.Logf("test case: %s", test.name) - t.Errorf("expected should provision %v but got %v\n", test.expectedShould, should) - } - } -} - -func TestShouldDelete(t *testing.T) { - tests := []struct { - name string - provisionerName string - volume *v1.PersistentVolume - serverGitVersion string - expectedShould bool - }{ - { - name: "should delete", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - serverGitVersion: "v1.5.0", - expectedShould: true, - }, - { - name: "1.4 and failed: should delete", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - serverGitVersion: "v1.4.0", - expectedShould: true, - }, - { - name: "1.5 and failed: shouldn't delete", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeFailed, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - serverGitVersion: "v1.5.0", - expectedShould: false, - }, - { - name: "volume still bound", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - serverGitVersion: "v1.5.0", - expectedShould: false, - }, - { - name: "non-delete reclaim policy", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimRetain, map[string]string{annDynamicallyProvisioned: "foo.bar/baz"}), - serverGitVersion: "v1.5.0", - expectedShould: false, - }, - { - name: "not this provisioner's job", - provisionerName: "foo.bar/baz", - volume: newVolume("volume-1", v1.VolumeReleased, v1.PersistentVolumeReclaimDelete, map[string]string{annDynamicallyProvisioned: "abc.def/ghi"}), - serverGitVersion: "v1.5.0", - expectedShould: false, - }, - } - for _, test := range tests { - client := fake.NewSimpleClientset() - provisioner := newTestProvisioner() - ctrl := newTestProvisionController(client, test.provisionerName, provisioner, test.serverGitVersion) - - should := ctrl.shouldDelete(test.volume) - if test.expectedShould != should { - t.Logf("test case: %s", test.name) - t.Errorf("expected should delete %v but got %v\n", test.expectedShould, should) - } - } -} - -func TestIsOnlyRecordUpdate(t *testing.T) { - tests := []struct { - name string - old *v1.PersistentVolumeClaim - new *v1.PersistentVolumeClaim - expectedIs bool - }{ - { - name: "is only record update", - old: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - new: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "b"}), - expectedIs: true, - }, - { - name: "is seen as only record update, stayed exactly the same", - old: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - new: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - expectedIs: true, - }, - { - name: "isn't only record update, class changed as well", - old: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - new: newClaim("claim-1", "1-1", "class-2", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "b"}), - expectedIs: false, - }, - { - name: "isn't only record update, only class changed", - old: newClaim("claim-1", "1-1", "class-1", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - new: newClaim("claim-1", "1-1", "class-2", "", map[string]string{rl.LeaderElectionRecordAnnotationKey: "a"}), - expectedIs: false, - }, - } - for _, test := range tests { - client := fake.NewSimpleClientset() - provisioner := newTestProvisioner() - ctrl := newTestProvisionController(client, "foo.bar/baz", provisioner, "v1.5.0") - - is, _ := ctrl.isOnlyRecordUpdate(test.old, test.new) - if test.expectedIs != is { - t.Logf("test case: %s", test.name) - t.Errorf("expected is only record update %v but got %v\n", test.expectedIs, is) - } - } -} - -func newTestProvisionController( - client kubernetes.Interface, - provisionerName string, - provisioner Provisioner, - serverGitVersion string, -) *ProvisionController { - ctrl := NewProvisionController( - client, - provisionerName, - provisioner, - serverGitVersion, - ResyncPeriod(resyncPeriod), - ExponentialBackOffOnError(false), - CreateProvisionedPVInterval(10*time.Millisecond), - LeaseDuration(2*resyncPeriod), - RenewDeadline(resyncPeriod), - RetryPeriod(resyncPeriod/2), - TermLimit(2*resyncPeriod)) - return ctrl -} - -func newStorageClass(name, provisioner string, reclaimPolicy v1.PersistentVolumeReclaimPolicy) *storagebeta.StorageClass { - return &storagebeta.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Provisioner: provisioner, - ReclaimPolicy: &reclaimPolicy, - } -} - -func newClaim(name, claimUID, provisioner, volumeName string, annotations map[string]string) *v1.PersistentVolumeClaim { - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: v1.NamespaceDefault, - UID: types.UID(claimUID), - ResourceVersion: "0", - Annotations: map[string]string{annClass: provisioner}, - SelfLink: "/api/v1/namespaces/" + v1.NamespaceDefault + "/persistentvolumeclaims/" + name, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Mi"), - }, - }, - VolumeName: volumeName, - }, - Status: v1.PersistentVolumeClaimStatus{ - Phase: v1.ClaimPending, - }, - } - for k, v := range annotations { - claim.Annotations[k] = v - } - return claim -} - -func newVolume(name string, phase v1.PersistentVolumePhase, policy v1.PersistentVolumeReclaimPolicy, annotations map[string]string) *v1.PersistentVolume { - pv := &v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Annotations: annotations, - SelfLink: "/api/v1/persistentvolumes/" + name, - }, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeReclaimPolicy: policy, - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}, - Capacity: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Mi"), - }, - PersistentVolumeSource: v1.PersistentVolumeSource{ - NFS: &v1.NFSVolumeSource{ - Server: "foo", - Path: "bar", - ReadOnly: false, - }, - }, - }, - Status: v1.PersistentVolumeStatus{ - Phase: phase, - }, - } - - return pv -} - -// newProvisionedVolume returns the volume the test controller should provision for the -// given claim with the given class -func newProvisionedVolume(storageClass *storagebeta.StorageClass, claim *v1.PersistentVolumeClaim) *v1.PersistentVolume { - // pv.Spec MUST be set to match requirements in claim.Spec, especially access mode and PV size. The provisioned volume size MUST NOT be smaller than size requested in the claim, however it MAY be larger. - options := VolumeOptions{ - PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, - PVName: "pvc-" + string(claim.ObjectMeta.UID), - PVC: claim, - Parameters: storageClass.Parameters, - } - volume, _ := newTestProvisioner().Provision(options) - - // pv.Spec.ClaimRef MUST point to the claim that led to its creation (including the claim UID). - v1.AddToScheme(scheme.Scheme) - volume.Spec.ClaimRef, _ = ref.GetReference(scheme.Scheme, claim) - - // pv.Annotations["pv.kubernetes.io/provisioned-by"] MUST be set to name of the external provisioner. This provisioner will be used to delete the volume. - // pv.Annotations["volume.beta.kubernetes.io/storage-class"] MUST be set to name of the storage class requested by the claim. - volume.Annotations = map[string]string{annDynamicallyProvisioned: storageClass.Provisioner, annClass: storageClass.Name} - - // TODO implement options.ProvisionerSelector parsing - // pv.Labels MUST be set to match claim.spec.selector. The provisioner MAY add additional labels. - - return volume -} - -func newTestProvisioner() *testProvisioner { - return &testProvisioner{make(chan bool, 16)} -} - -type testProvisioner struct { - provisionCalls chan bool -} - -var _ Provisioner = &testProvisioner{} - -func (p *testProvisioner) Provision(options VolumeOptions) (*v1.PersistentVolume, error) { - p.provisionCalls <- true - - // Sleep to simulate work done by Provision...for long enough that - // TestMultipleControllers will consistently fail with lock disabled. If - // Provision happens too fast, the first controller creates the PV too soon - // and the next controllers won't call Provision even though they're clearly - // racing when there's no lock - time.Sleep(50 * time.Millisecond) - - pv := &v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: options.PVName, - }, - Spec: v1.PersistentVolumeSpec{ - PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy, - AccessModes: options.PVC.Spec.AccessModes, - Capacity: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)], - }, - PersistentVolumeSource: v1.PersistentVolumeSource{ - NFS: &v1.NFSVolumeSource{ - Server: "foo", - Path: "bar", - ReadOnly: false, - }, - }, - }, - } - - return pv, nil -} - -func (p *testProvisioner) Delete(volume *v1.PersistentVolume) error { - return nil -} - -func newBadTestProvisioner() Provisioner { - return &badTestProvisioner{} -} - -type badTestProvisioner struct { -} - -var _ Provisioner = &badTestProvisioner{} - -func (p *badTestProvisioner) Provision(options VolumeOptions) (*v1.PersistentVolume, error) { - return nil, errors.New("fake error") -} - -func (p *badTestProvisioner) Delete(volume *v1.PersistentVolume) error { - return errors.New("fake error") -} - -func newIgnoredProvisioner() Provisioner { - return &ignoredProvisioner{} -} - -type ignoredProvisioner struct { -} - -var _ Provisioner = &ignoredProvisioner{} - -func (i *ignoredProvisioner) Provision(options VolumeOptions) (*v1.PersistentVolume, error) { - if options.PVC.Name == "claim-2" { - return nil, &IgnoredError{"Ignored"} - } - - return newProvisionedVolume(newStorageClass("class-1", "foo.bar/baz", options.PersistentVolumeReclaimPolicy), newClaim("claim-1", "uid-1-1", "class-1", "", nil)), nil -} - -func (i *ignoredProvisioner) Delete(volume *v1.PersistentVolume) error { - return nil -} - -type claimReactor struct { - fake *fakev1core.FakeCoreV1 - claims map[string]*v1.PersistentVolumeClaim - lock sync.Mutex - claimSource *fcache.FakePVCControllerSource -} - -func (r *claimReactor) React(action testclient.Action) (handled bool, ret runtime.Object, err error) { - r.lock.Lock() - defer r.lock.Unlock() - switch { - case action.Matches("update", "persistentvolumeclaims"): - obj := action.(testclient.UpdateAction).GetObject() - - claim := obj.(*v1.PersistentVolumeClaim) - - // Check and bump object version - storedClaim, found := r.claims[claim.Name] - if found { - storedVer, _ := strconv.Atoi(storedClaim.ResourceVersion) - requestedVer, _ := strconv.Atoi(claim.ResourceVersion) - if storedVer != requestedVer { - return true, obj, errors.New("VersionError") - } - claim.ResourceVersion = strconv.Itoa(storedVer + 1) - } else { - return true, nil, fmt.Errorf("Cannot update claim %s: claim not found", claim.Name) - } - - r.claims[claim.Name] = claim - r.claimSource.Modify(claim) - return true, claim, nil - case action.Matches("get", "persistentvolumeclaims"): - name := action.(testclient.GetAction).GetName() - claim, found := r.claims[name] - if found { - claimClone := claim.DeepCopy() - return true, claimClone, nil - } - return true, nil, fmt.Errorf("Cannot find claim %s", name) - } - - return false, nil, nil -} diff --git a/pkg/operator/ceph/provisioner/controller/leaderelection/leaderelection.go b/pkg/operator/ceph/provisioner/controller/leaderelection/leaderelection.go deleted file mode 100644 index 148a3a073..000000000 --- a/pkg/operator/ceph/provisioner/controller/leaderelection/leaderelection.go +++ /dev/null @@ -1,289 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This is a modified version of kube's leaderelection package which uses a -// dummy endpoints object & its annotation as a lock. Here a pvc is used and -// the lock is to help ensure only one provisioner (the leader) is trying to -// provision a volume for the pvc at a time. So the election lasts only until -// the task is completed. Adds also a 'TermLimit.' -// https://github.com/kubernetes/kubernetes/tree/release-1.5/pkg/client/leaderelection - -package leaderelection - -import ( - "fmt" - "reflect" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - - rl "github.com/rook/rook/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/golang/glog" -) - -const ( - // JitterFactor is the factor to jitter the RetryPeriod by - JitterFactor = 1.2 -) - -// NewLeaderElector creates a LeaderElector from a Config -func NewLeaderElector(lec Config) (*LeaderElector, error) { - if lec.LeaseDuration <= lec.RenewDeadline { - return nil, fmt.Errorf("leaseDuration must be greater than renewDeadline") - } - if lec.RenewDeadline <= time.Duration(JitterFactor*float64(lec.RetryPeriod)) { - return nil, fmt.Errorf("renewDeadline must be greater than retryPeriod*JitterFactor") - } - if lec.Lock == nil { - return nil, fmt.Errorf("lock must not be nil") - } - return &LeaderElector{ - config: lec, - }, nil -} - -// Config is a configuration for leader election -type Config struct { - // Lock is the resource that will be used for locking - Lock rl.Interface - - // LeaseDuration is the duration that non-leader candidates will - // wait to force acquire leadership. This is measured against time of - // last observed ack. - LeaseDuration time.Duration - // RenewDeadline is the duration that the acting master will retry - // refreshing leadership before giving up. - RenewDeadline time.Duration - // RetryPeriod is the duration the LeaderElector clients should wait - // between tries of actions. - RetryPeriod time.Duration - // TermLimit is the maximum duration that a leader may remain the leader - // to complete the task before it must give up its leadership. 0 for forever - // or indefinite. - TermLimit time.Duration - - // Callbacks are callbacks that are triggered during certain lifecycle - // events of the LeaderElector - Callbacks LeaderCallbacks -} - -// LeaderCallbacks are callbacks that are triggered during certain -// lifecycle events of the LeaderElector. These are invoked asynchronously. -// -// possible future callbacks: -// * OnChallenge() -type LeaderCallbacks struct { - // OnStartedLeading is called when a LeaderElector client starts leading - OnStartedLeading func(stop <-chan struct{}) - // OnStoppedLeading is called when a LeaderElector client stops leading - OnStoppedLeading func() - // OnNewLeader is called when the client observes a leader that is - // not the previously observed leader. This includes the first observed - // leader when the client starts. - OnNewLeader func(identity string) -} - -// LeaderElector is a leader election client. -// -// possible future methods: -// * (le *LeaderElector) IsLeader() -// * (le *LeaderElector) GetLeader() -type LeaderElector struct { - config Config - // internal bookkeeping - observedRecord rl.LeaderElectionRecord - observedTime time.Time - // used to implement OnNewLeader(), may lag slightly from the - // value observedRecord.HolderIdentity if the transition has - // not yet been reported. - reportedLeader string -} - -// Run starts the leader election loop -func (le *LeaderElector) Run(task <-chan bool) { - defer func() { - runtime.HandleCrash() - }() - over := le.acquire(task) - if over { - return - } - stop := make(chan struct{}) - go le.config.Callbacks.OnStartedLeading(stop) - timeout := make(chan bool, 1) - go func() { - time.Sleep(le.config.TermLimit) - timeout <- true - }() - le.renew(task, timeout) - close(stop) - le.config.Callbacks.OnStoppedLeading() -} - -// GetLeader returns the identity of the last observed leader or returns the empty string if -// no leader has yet been observed. -func (le *LeaderElector) GetLeader() string { - return le.observedRecord.HolderIdentity -} - -// IsLeader returns true if the last observed leader was this client else returns false. -func (le *LeaderElector) IsLeader() bool { - return le.observedRecord.HolderIdentity == le.config.Lock.Identity() -} - -// acquire loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew succeeds -// or the task has successfully finished in which case there is no longer a need to acquire -func (le *LeaderElector) acquire(task <-chan bool) bool { - over := false - stop := make(chan struct{}) - glog.Infof("attempting to acquire leader lease...") - wait.JitterUntil(func() { - select { - case taskSucceeded := <-task: - if taskSucceeded { - // if the leader succeeded at the task, stop trying to acquire - desc := le.config.Lock.Describe() - glog.Infof("stopped trying to acquire lease %v, task succeeded", desc) - over = true - close(stop) - return - } - default: - } - succeeded := le.tryAcquireOrRenew() - le.maybeReportTransition() - desc := le.config.Lock.Describe() - if !succeeded { - glog.V(4).Infof("failed to acquire lease %v", desc) - return - } - le.config.Lock.RecordEvent("became leader") - glog.Infof("successfully acquired lease %v", desc) - close(stop) - }, le.config.RetryPeriod, JitterFactor, true, stop) - - return over -} - -// renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails -// or the task has either succeeded or failed in which case leadership must be given up. -func (le *LeaderElector) renew(task <-chan bool, timeout <-chan bool) { - stop := make(chan struct{}) - wait.Until(func() { - select { - case taskSucceeded := <-task: - // if the leader (us) either succeeded or failed at the task, stop trying to renew - desc := le.config.Lock.Describe() - taskSucceededStr := "succeeded" - if !taskSucceeded { - taskSucceededStr = "failed" - } - glog.Infof("stopped trying to renew lease %v, task %s", desc, taskSucceededStr) - close(stop) - return - case <-timeout: - // our term limit has ended, let somebody else have a try - desc := le.config.Lock.Describe() - glog.Infof("stopped trying to renew lease %v, timeout reached", desc) - close(stop) - return - default: - } - err := wait.Poll(le.config.RetryPeriod, le.config.RenewDeadline, func() (bool, error) { - return le.tryAcquireOrRenew(), nil - }) - le.maybeReportTransition() - desc := le.config.Lock.Describe() - if err == nil { - glog.V(4).Infof("successfully renewed lease %v", desc) - return - } - le.config.Lock.RecordEvent("stopped leading") - glog.Infof("failed to renew lease %v", desc) - close(stop) - }, 0, stop) -} - -// tryAcquireOrRenew tries to acquire a leader lease if it is not already acquired, -// else it tries to renew the lease if it has already been acquired. Returns true -// on success else returns false. -func (le *LeaderElector) tryAcquireOrRenew() bool { - now := metav1.Now() - leaderElectionRecord := rl.LeaderElectionRecord{ - HolderIdentity: le.config.Lock.Identity(), - LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second), - RenewTime: now, - AcquireTime: now, - } - - // 1. obtain or create the ElectionRecord - oldLeaderElectionRecord, err := le.config.Lock.Get() - if err != nil { - if !errors.IsNotFound(err) { - glog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err) - return false - } - if err = le.config.Lock.Create(leaderElectionRecord); err != nil { - glog.Errorf("error initially creating leader election record: %v", err) - return false - } - le.observedRecord = leaderElectionRecord - le.observedTime = time.Now() - return true - } - - // 2. Record obtained, check the Identity & Time - if !reflect.DeepEqual(le.observedRecord, *oldLeaderElectionRecord) { - le.observedRecord = *oldLeaderElectionRecord - le.observedTime = time.Now() - } - if le.observedTime.Add(le.config.LeaseDuration).After(now.Time) && - oldLeaderElectionRecord.HolderIdentity != le.config.Lock.Identity() { - glog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) - return false - } - - // 3. We're going to try to update. The leaderElectionRecord is set to it's default - // here. Let's correct it before updating. - if oldLeaderElectionRecord.HolderIdentity == le.config.Lock.Identity() { - leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime - } else { - leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1 - } - - // update the lock itself - if err = le.config.Lock.Update(leaderElectionRecord); err != nil { - glog.Errorf("Failed to update lock: %v", err) - return false - } - le.observedRecord = leaderElectionRecord - le.observedTime = time.Now() - return true -} - -func (le *LeaderElector) maybeReportTransition() { - if le.observedRecord.HolderIdentity == le.reportedLeader { - return - } - le.reportedLeader = le.observedRecord.HolderIdentity - if le.config.Callbacks.OnNewLeader != nil { - go le.config.Callbacks.OnNewLeader(le.reportedLeader) - } -} diff --git a/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/interface.go b/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/interface.go deleted file mode 100644 index 9a8c760da..000000000 --- a/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/interface.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resourcelock - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" -) - -const ( - // LeaderElectionRecordAnnotationKey is the annotation key for records - LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" -) - -// LeaderElectionRecord is the record that is stored in the leader election annotation. -// This information should be used for observational purposes only and could be replaced -// with a random string (e.g. UUID) with only slight modification of this code. -// TODO(mikedanese): this should potentially be versioned -type LeaderElectionRecord struct { - HolderIdentity string `json:"holderIdentity"` - LeaseDurationSeconds int `json:"leaseDurationSeconds"` - AcquireTime metav1.Time `json:"acquireTime"` - RenewTime metav1.Time `json:"renewTime"` - LeaderTransitions int `json:"leaderTransitions"` -} - -// Config common data that exists across different -// resource locks -type Config struct { - Identity string - EventRecorder record.EventRecorder -} - -// Interface offers a common interface for locking on arbitrary -// resources used in leader election. The Interface is used -// to hide the details on specific implementations in order to allow -// them to change over time. This interface is strictly for use -// by the leaderelection code. -type Interface interface { - // Get returns the LeaderElectionRecord - Get() (*LeaderElectionRecord, error) - - // Create attempts to create a LeaderElectionRecord - Create(ler LeaderElectionRecord) error - - // Update will update and existing LeaderElectionRecord - Update(ler LeaderElectionRecord) error - - // RecordEvent is used to record events - RecordEvent(string) - - // Identity will return the locks Identity - Identity() string - - // Describe is used to convert details on current resource lock - // into a string - Describe() string -} diff --git a/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/provisionpvclock.go b/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/provisionpvclock.go deleted file mode 100644 index 5b5ca30b7..000000000 --- a/pkg/operator/ceph/provisioner/controller/leaderelection/resourcelock/provisionpvclock.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resourcelock - -import ( - "encoding/json" - "errors" - "fmt" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" -) - -// ProvisionPVCLock is a lock on an existing PVC to provision a PV for -type ProvisionPVCLock struct { - // PVCMeta should contain a Name and a Namespace of a PVC - // object that the LeaderElector will attempt to lead. - PVCMeta metav1.ObjectMeta - Client clientset.Interface - LockConfig Config - p *v1.PersistentVolumeClaim -} - -// Get returns the LeaderElectionRecord -func (pl *ProvisionPVCLock) Get() (*LeaderElectionRecord, error) { - var record LeaderElectionRecord - var err error - pl.p, err = pl.Client.CoreV1().PersistentVolumeClaims(pl.PVCMeta.Namespace).Get(pl.PVCMeta.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - // TODO there should be a way to give up if the pvc is already bound...we are doing a Get regardless - if pl.p.Annotations == nil { - pl.p.Annotations = make(map[string]string) - } - if recordBytes, found := pl.p.Annotations[LeaderElectionRecordAnnotationKey]; found { - if err := json.Unmarshal([]byte(recordBytes), &record); err != nil { - return nil, err - } - } - return &record, nil -} - -// Create is not allowed, the PVC should already exist -func (pl *ProvisionPVCLock) Create(ler LeaderElectionRecord) error { - return errors.New("create not allowed, PVC should already exist") -} - -// Update will update and existing annotation on a given resource. -func (pl *ProvisionPVCLock) Update(ler LeaderElectionRecord) error { - if pl.p == nil { - return errors.New("PVC not initialized, call get first") - } - recordBytes, err := json.Marshal(ler) - if err != nil { - return err - } - pl.p.Annotations[LeaderElectionRecordAnnotationKey] = string(recordBytes) - pl.p, err = pl.Client.CoreV1().PersistentVolumeClaims(pl.PVCMeta.Namespace).Update(pl.p) - return err -} - -// RecordEvent in leader election while adding meta-data -func (pl *ProvisionPVCLock) RecordEvent(s string) { - // events := fmt.Sprintf("%v %v", pl.LockConfig.Identity, s) - // pl.LockConfig.EventRecorder.Eventf(&v1.PersistentVolumeClaim{ObjectMeta: pl.p.ObjectMeta}, v1.EventTypeNormal, "LeaderElection", events) -} - -// Describe is used to convert details on current resource lock -// into a string -func (pl *ProvisionPVCLock) Describe() string { - return fmt.Sprintf("to provision for pvc %v/%v", pl.PVCMeta.Namespace, pl.PVCMeta.Name) -} - -// Identity returns the Identity of the lock -func (pl *ProvisionPVCLock) Identity() string { - return pl.LockConfig.Identity -} diff --git a/pkg/operator/ceph/provisioner/controller/volume.go b/pkg/operator/ceph/provisioner/controller/volume.go deleted file mode 100644 index 1522cb148..000000000 --- a/pkg/operator/ceph/provisioner/controller/volume.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - - "k8s.io/api/core/v1" -) - -// Provisioner is an interface that creates templates for PersistentVolumes -// and can create the volume as a new resource in the infrastructure provider. -// It can also remove the volume it created from the underlying storage -// provider. -type Provisioner interface { - // Provision creates a volume i.e. the storage asset and returns a PV object - // for the volume - Provision(VolumeOptions) (*v1.PersistentVolume, error) - // Delete removes the storage asset that was created by Provision backing the - // given PV. Does not delete the PV object itself. - // - // May return IgnoredError to indicate that the call has been ignored and no - // action taken. - Delete(*v1.PersistentVolume) error -} - -// IgnoredError is the value for Delete to return to indicate that the call has -// been ignored and no action taken. In case multiple provisioners are serving -// the same storage class, provisioners may ignore PVs they are not responsible -// for (e.g. ones they didn't create). The controller will act accordingly, -// i.e. it won't emit a misleading VolumeFailedDelete event. -type IgnoredError struct { - Reason string -} - -func (e *IgnoredError) Error() string { - return fmt.Sprintf("ignored because %s", e.Reason) -} - -// VolumeOptions contains option information about a volume -// https://github.com/kubernetes/kubernetes/blob/release-1.4/pkg/volume/plugins.go -type VolumeOptions struct { - // Reclamation policy for a persistent volume - PersistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy - // PV.Name of the appropriate PersistentVolume. Used to generate cloud - // volume name. - PVName string - // PVC is reference to the claim that lead to provisioning of a new PV. - // Provisioners *must* create a PV that would be matched by this PVC, - // i.e. with required capacity, accessMode, labels matching PVC.Selector and - // so on. - PVC *v1.PersistentVolumeClaim - // Volume provisioning parameters from StorageClass - Parameters map[string]string -} diff --git a/pkg/operator/ceph/provisioner/provisioner.go b/pkg/operator/ceph/provisioner/provisioner.go index 353e670e6..ba878e8f4 100644 --- a/pkg/operator/ceph/provisioner/provisioner.go +++ b/pkg/operator/ceph/provisioner/provisioner.go @@ -22,11 +22,11 @@ import ( "strings" "github.com/coreos/pkg/capnslog" + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume" ceph "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/cluster" - "github.com/rook/rook/pkg/operator/ceph/provisioner/controller" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/operator/ceph/provisioner/provisioner_test.go b/pkg/operator/ceph/provisioner/provisioner_test.go index 65e79cba9..3157d9ee2 100644 --- a/pkg/operator/ceph/provisioner/provisioner_test.go +++ b/pkg/operator/ceph/provisioner/provisioner_test.go @@ -24,9 +24,9 @@ import ( "strings" "testing" + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" "github.com/rook/rook/pkg/clusterd" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" - "github.com/rook/rook/pkg/operator/ceph/provisioner/controller" "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 50d2eadc6..1bca5838e 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -328,6 +328,7 @@ rules: - events - persistentvolumes - persistentvolumeclaims + - endpoints verbs: - get - list diff --git a/tests/integration/upgrade_test.go b/tests/integration/upgrade_test.go index ee410704b..c5abf5a19 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/upgrade_test.go @@ -107,6 +107,10 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { assert.Nil(s.T(), s.k8sh.WriteToPod(s.namespace, filePodName, preFilename, message)) assert.Nil(s.T(), s.k8sh.ReadFromPod(s.namespace, filePodName, preFilename, message)) + // Update to the next version of cluster roles before the operator is restarted + err = s.updateClusterRoles() + require.Nil(s.T(), err) + // Upgrade to master require.Nil(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) @@ -145,3 +149,80 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { assert.Nil(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5)) assert.Nil(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5)) } + +// Update the clusterroles that have been modified in master from the previous release +func (s *UpgradeSuite) updateClusterRoles() error { + if _, err := s.k8sh.DeleteResource("ClusterRole", "rook-ceph-global"); err != nil { + return err + } + + newResources := ` +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +` + logger.Infof("creating the new resources that have been added since 0.9") + _, err := s.k8sh.ResourceOperation("create", newResources) + return err +}