cassandra: sidecar implementation
Implementation of the cassandra sidecar logic. This sidecar helps us inject our custom values during startup and provides HTTP liveness and readiness checks. In the future it will also be used for: - Shutting Down / Decommissioning a cassandra instance - Backups - Restores Signed-off-by: Yannis Zarkadas <yanniszarkadas@gmail.com>
This commit is contained in:
1 parent
c8c38d8584
commit
367d485407
4 files changed
+769
No files matched your search
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cassandra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rook/rook/cmd/rook/rook"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/sidecar"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
"k8s.io/apiserver/pkg/server"
|
||||
"os"
|
||||
)
|
||||
|
||||
var sidecarCmd = &cobra.Command{
|
||||
Use: "sidecar",
|
||||
Short: "Runs the cassandra sidecar to deploy and manage cassandra in Kubernetes",
|
||||
Long: `Runs the cassandra sidecar to deploy and manage cassandra in kubernetes clusters.
|
||||
https://github.com/rook/rook`,
|
||||
}
|
||||
|
||||
func init() {
|
||||
flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
|
||||
sidecarCmd.RunE = startSidecar
|
||||
}
|
||||
|
||||
func startSidecar(cmd *cobra.Command, args []string) error {
|
||||
rook.SetLogLevel()
|
||||
rook.LogStartupInfo(operatorCmd.Flags())
|
||||
|
||||
kubeClient, _, rookClient, err := rook.GetClientset()
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err))
|
||||
}
|
||||
|
||||
podName := os.Getenv(k8sutil.PodNameEnvVar)
|
||||
if podName == "" {
|
||||
rook.TerminateFatal(fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file"))
|
||||
}
|
||||
podNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
if podNamespace == "" {
|
||||
rook.TerminateFatal(fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file"))
|
||||
}
|
||||
|
||||
mc, err := sidecar.New(
|
||||
podName,
|
||||
podNamespace,
|
||||
kubeClient,
|
||||
rookClient,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to initialize member controller: %s", err.Error()))
|
||||
}
|
||||
logger.Infof("Initialized Member Controller: %+v", mc)
|
||||
|
||||
// Create a channel to receive OS signals
|
||||
stopCh := server.SetupSignalHandler()
|
||||
|
||||
// Start the controller loop
|
||||
logger.Infof("Starting rook sidecar for Cassandra.")
|
||||
if err = mc.Run(1, stopCh); err != nil {
|
||||
logger.Fatalf("Error running sidecar: %s", err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package sidecar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/yanniszark/go-nodetool/nodetool"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// setupHTTPChecks brings up the liveness and readiness probes
|
||||
func (m *MemberController) setupHTTPChecks() error {
|
||||
|
||||
http.HandleFunc(constants.LivenessProbePath, livenessCheck(m))
|
||||
http.HandleFunc(constants.ReadinessProbePath, readinessCheck(m))
|
||||
|
||||
err := http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", constants.ProbePort), nil)
|
||||
// If ListenAndServe returns, something went wrong
|
||||
m.logger.Fatalf("Error in HTTP checks: %s", err.Error())
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
func livenessCheck(m *MemberController) func(http.ResponseWriter, *http.Request) {
|
||||
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
status := http.StatusOK
|
||||
|
||||
// Check if JMX is reachable
|
||||
_, err := m.nodetool.Status()
|
||||
if err != nil {
|
||||
m.logger.Errorf("Liveness check failed with error: %s", err.Error())
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
w.WriteHeader(status)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func readinessCheck(m *MemberController) func(http.ResponseWriter, *http.Request) {
|
||||
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
status := http.StatusOK
|
||||
|
||||
err := func() error {
|
||||
// Contact Cassandra to learn about the status of the member
|
||||
HostIDMap, err := m.nodetool.Status()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error while executing nodetool status in readiness check: %s", err.Error())
|
||||
}
|
||||
// Get local node through static ip
|
||||
localNode, ok := HostIDMap[m.ip]
|
||||
if !ok {
|
||||
return fmt.Errorf("Couldn't find node with ip %s in nodetool status.", m.ip)
|
||||
}
|
||||
// Check local node status
|
||||
// Up means the member is alive
|
||||
if localNode.Status != nodetool.NodeStatusUp {
|
||||
return fmt.Errorf("Unexpected local node status: %s", localNode.Status)
|
||||
}
|
||||
// Check local node state
|
||||
// Normal means that the member has completed bootstrap and joined the cluster
|
||||
if localNode.State != nodetool.NodeStateNormal {
|
||||
return fmt.Errorf("Unexpected local node state: %s", localNode.State)
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
m.logger.Errorf("Readiness check failed with error: %s", err.Error())
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
w.WriteHeader(status)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package sidecar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/ghodss/yaml"
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/controller/util"
|
||||
"io/ioutil"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// Cassandra-Specific
|
||||
configDirCassandra = "/etc/cassandra"
|
||||
cassandraYAMLPath = configDirCassandra + "/" + "cassandra.yaml"
|
||||
cassandraEnvPath = configDirCassandra + "/" + "cassandra-env.sh"
|
||||
cassandraRackDCPropertiesPath = configDirCassandra + "/" + "cassandra-rackdc.properties"
|
||||
|
||||
// Scylla-Specific
|
||||
configDirScylla = "/etc/scylla"
|
||||
scyllaYAMLPath = configDirScylla + "/" + "scylla.yaml"
|
||||
scyllaRackDCPropertiesPath = configDirScylla + "/" + "cassandra-rackdc.properties"
|
||||
scyllaJMXPath = "/usr/lib/scylla/jmx/scylla-jmx"
|
||||
|
||||
// Common
|
||||
jolokiaPath = constants.PluginDirName + "/" + "jolokia.jar"
|
||||
entrypointPath = "/entrypoint.sh"
|
||||
rackDCPropertiesFormat = "dc=%s" + "\n" + "rack=%s" + "\n" + "prefer_local=false" + "\n"
|
||||
)
|
||||
|
||||
// generateConfigFiles injects the default configuration files
|
||||
// with our custom values.
|
||||
func (m *MemberController) generateConfigFiles() error {
|
||||
|
||||
var err error
|
||||
m.logger.Info("Generating config files")
|
||||
|
||||
if m.mode == cassandrav1alpha1.ClusterModeScylla {
|
||||
err = m.generateScyllaConfigFiles()
|
||||
} else {
|
||||
err = m.generateCassandraConfigFiles()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// generateCassandraConfigFiles generates the necessary config files for Cassandra.
|
||||
// Currently, those are:
|
||||
// - cassandra.yaml
|
||||
// - cassandra-env.sh
|
||||
// - cassandra-rackdc.properties
|
||||
// - entrypoint-sh
|
||||
func (m *MemberController) generateCassandraConfigFiles() error {
|
||||
|
||||
/////////////////////////////
|
||||
// Generate cassandra.yaml //
|
||||
/////////////////////////////
|
||||
|
||||
// Read default cassandra.yaml
|
||||
cassandraYAML, err := ioutil.ReadFile(cassandraYAMLPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected error trying to open cassandra.yaml: %s", err.Error())
|
||||
}
|
||||
|
||||
customCassandraYAML, err := m.overrideConfigValues(cassandraYAML)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to override config values: %s", err.Error())
|
||||
}
|
||||
|
||||
// Write result to file
|
||||
if err = ioutil.WriteFile(cassandraYAMLPath, customCassandraYAML, os.ModePerm); err != nil {
|
||||
m.logger.Errorf("error trying to write cassandra.yaml: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
// Generate cassandra-rackdc.properties //
|
||||
//////////////////////////////////////////
|
||||
|
||||
rackdcProperties := []byte(fmt.Sprintf(rackDCPropertiesFormat, m.datacenter, m.rack))
|
||||
if err = ioutil.WriteFile(cassandraRackDCPropertiesPath, rackdcProperties, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error trying to write cassandra-rackdc.properties: %s", err.Error())
|
||||
}
|
||||
|
||||
/////////////////////////////////////////
|
||||
// Generate cassandra-env.sh //
|
||||
/////////////////////////////////////////
|
||||
|
||||
cassandraEnv, err := ioutil.ReadFile(cassandraEnvPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to open cassandra-env.sh, %s", err.Error())
|
||||
}
|
||||
|
||||
// Calculate heap sizes
|
||||
// https://github.com/apache/cassandra/blob/521542ff26f9482b733e4f0f86281f07c3af29da/conf/cassandra-env.sh
|
||||
cpu := os.Getenv(constants.ResourceLimitCPUEnvVar)
|
||||
if cpu == "" {
|
||||
return fmt.Errorf("%s env variable not found", constants.ResourceLimitCPUEnvVar)
|
||||
}
|
||||
cpuNumber, _ := strconv.ParseInt(cpu, 10, 64)
|
||||
mem := os.Getenv(constants.ResourceLimitMemoryEnvVar)
|
||||
if mem == "" {
|
||||
return fmt.Errorf("%s env variable not found", constants.ResourceLimitMemoryEnvVar)
|
||||
}
|
||||
memNumber, _ := strconv.ParseInt(mem, 10, 64)
|
||||
maxHeapSize := util.Max(util.Min(memNumber/2, 1024), util.Min(memNumber/4, 8192))
|
||||
heapNewSize := util.Min(maxHeapSize/4, 100*cpuNumber)
|
||||
if err := os.Setenv("MAX_HEAP_SIZE", fmt.Sprintf("%dM", maxHeapSize)); err != nil {
|
||||
return fmt.Errorf("error setting MAX_HEAP_SIZE: %s", err.Error())
|
||||
}
|
||||
if err := os.Setenv("HEAP_NEWSIZE", fmt.Sprintf("%dM", heapNewSize)); err != nil {
|
||||
return fmt.Errorf("error setting HEAP_NEWSIZE: %s", err.Error())
|
||||
}
|
||||
|
||||
// Add jolokia javaagent
|
||||
jolokiaConfig := []byte(fmt.Sprintf(`JVM_OPTS="$JVM_OPTS %s"`,
|
||||
getJolokiaConfig()))
|
||||
|
||||
err = ioutil.WriteFile(cassandraEnvPath, append(cassandraEnv, jolokiaConfig...), os.ModePerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to write cassandra-env.sh: %s", err.Error())
|
||||
}
|
||||
|
||||
////////////////////////////
|
||||
// Generate entrypoint.sh //
|
||||
////////////////////////////
|
||||
|
||||
entrypoint := "#!/bin/sh" + "\n" + "exec cassandra -f -R"
|
||||
if err := ioutil.WriteFile(entrypointPath, []byte(entrypoint), os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error trying to write cassandra entrypoint: %s", err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// generateScyllaConfigFiles generates the necessary config files for Scylla.
|
||||
// Currently, those are:
|
||||
// - scylla.yaml
|
||||
// - cassandra-rackdc.properties
|
||||
// - scylla-jmx
|
||||
// - entrypoint.sh
|
||||
func (m *MemberController) generateScyllaConfigFiles() error {
|
||||
|
||||
// TODO: remove scylla.yaml gen once the entrypoint script in scylla gets
|
||||
// the necessary options
|
||||
|
||||
/////////////////////////////
|
||||
// Generate scylla.yaml //
|
||||
/////////////////////////////
|
||||
|
||||
// Read default scylla.yaml
|
||||
scyllaYAML, err := ioutil.ReadFile(scyllaYAMLPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected error trying to open scylla.yaml: %s", err.Error())
|
||||
}
|
||||
|
||||
customScyllaYAML, err := m.overrideConfigValues(scyllaYAML)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error trying to override config values: %s", err.Error())
|
||||
}
|
||||
|
||||
// Write result to file
|
||||
if err = ioutil.WriteFile(scyllaYAMLPath, customScyllaYAML, os.ModePerm); err != nil {
|
||||
m.logger.Errorf("error trying to write scylla.yaml: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
// Generate cassandra-rackdc.properties //
|
||||
//////////////////////////////////////////
|
||||
|
||||
rackdcProperties := []byte(fmt.Sprintf(rackDCPropertiesFormat, m.datacenter, m.rack))
|
||||
if err := ioutil.WriteFile(scyllaRackDCPropertiesPath, rackdcProperties, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error trying to write cassandra-rackdc.properties: %s", err.Error())
|
||||
}
|
||||
|
||||
/////////////////////////////////////////
|
||||
// Edit scylla-jmx with jolokia option //
|
||||
/////////////////////////////////////////
|
||||
|
||||
scyllaJMXBytes, err := ioutil.ReadFile(scyllaJMXPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading scylla-jmx: %s", err.Error())
|
||||
}
|
||||
scyllaJMX := string(scyllaJMXBytes)
|
||||
splitIndex := strings.Index(scyllaJMX, `\`) + len(`\`)
|
||||
m.logger.Infof("Split index = %d", splitIndex)
|
||||
injectedLine := fmt.Sprintf("\n %s \\", getJolokiaConfig())
|
||||
scyllaJMXCustom := scyllaJMX[:splitIndex] + injectedLine + scyllaJMX[splitIndex:]
|
||||
if err := ioutil.WriteFile(scyllaJMXPath, []byte(scyllaJMXCustom), os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error writing scylla-jmx: %s", err.Error())
|
||||
}
|
||||
|
||||
////////////////////////////
|
||||
// Generate entrypoint.sh //
|
||||
////////////////////////////
|
||||
|
||||
entrypoint, err := m.scyllaEntrypoint()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating scylla entrypoint: %s", err.Error())
|
||||
}
|
||||
|
||||
m.logger.Infof("Scylla entrypoint script:\n %s", entrypoint)
|
||||
if err := ioutil.WriteFile(entrypointPath, []byte(entrypoint), os.ModePerm); err != nil {
|
||||
return fmt.Errorf("error trying to write scylla entrypoint: %s", err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// scyllaEntrypoint returns the entrypoint script for scylla
|
||||
func (m *MemberController) scyllaEntrypoint() (string, error) {
|
||||
|
||||
// Get seeds
|
||||
seeds, err := m.getSeeds()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting seeds: %s", err.Error())
|
||||
}
|
||||
|
||||
// Get local ip
|
||||
localIP := os.Getenv(constants.PodIPEnvVar)
|
||||
if localIP == "" {
|
||||
return "", fmt.Errorf("POD_IP environment variable not set")
|
||||
}
|
||||
|
||||
// See if we need to run in developer mode
|
||||
devMode := "0"
|
||||
c, err := m.rookClient.CassandraV1alpha1().Clusters(m.namespace).Get(m.cluster, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting cluster: %s", err.Error())
|
||||
}
|
||||
if val, ok := c.Annotations[constants.DeveloperModeAnnotation]; ok && val == constants.LabelValueTrue {
|
||||
devMode = "1"
|
||||
}
|
||||
|
||||
// Get cpu cores
|
||||
cpu := os.Getenv(constants.ResourceLimitCPUEnvVar)
|
||||
if cpu == "" {
|
||||
return "", fmt.Errorf("%s env variable not found", constants.ResourceLimitCPUEnvVar)
|
||||
}
|
||||
|
||||
// Get memory
|
||||
mem := os.Getenv(constants.ResourceLimitMemoryEnvVar)
|
||||
if mem == "" {
|
||||
return "", fmt.Errorf("%s env variable not found", constants.ResourceLimitMemoryEnvVar)
|
||||
}
|
||||
// Leave some memory for other stuff
|
||||
memNumber, _ := strconv.ParseInt(mem, 10, 64)
|
||||
mem = fmt.Sprintf("%dM", util.Max(memNumber-700, 0))
|
||||
|
||||
opts := []struct {
|
||||
flag, value string
|
||||
}{
|
||||
{
|
||||
flag: "listen-address",
|
||||
value: localIP,
|
||||
},
|
||||
{
|
||||
flag: "broadcast-address",
|
||||
value: m.ip,
|
||||
},
|
||||
{
|
||||
flag: "broadcast-rpc-address",
|
||||
value: m.ip,
|
||||
},
|
||||
{
|
||||
flag: "seeds",
|
||||
value: seeds,
|
||||
},
|
||||
{
|
||||
flag: "developer-mode",
|
||||
value: devMode,
|
||||
},
|
||||
{
|
||||
flag: "smp",
|
||||
value: cpu,
|
||||
},
|
||||
{
|
||||
flag: "memory",
|
||||
value: mem,
|
||||
},
|
||||
}
|
||||
|
||||
entrypoint := "#!/bin/sh" + "\n" + "exec /docker-entrypoint.py"
|
||||
for _, opt := range opts {
|
||||
entrypoint = fmt.Sprintf("%s --%s %s", entrypoint, opt.flag, opt.value)
|
||||
}
|
||||
return entrypoint, nil
|
||||
}
|
||||
|
||||
// overrideConfigValues overrides the default config values with
|
||||
// our custom values, for the fields that are of interest to us
|
||||
func (m *MemberController) overrideConfigValues(configText []byte) ([]byte, error) {
|
||||
|
||||
var config map[string]interface{}
|
||||
|
||||
if err := yaml.Unmarshal(configText, &config); err != nil {
|
||||
return nil, fmt.Errorf("error unmarshaling cassandra.yaml: %s", err.Error())
|
||||
}
|
||||
|
||||
seeds, err := m.getSeeds()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting seeds: %s", err.Error())
|
||||
}
|
||||
|
||||
localIP := os.Getenv(constants.PodIPEnvVar)
|
||||
if localIP == "" {
|
||||
return nil, fmt.Errorf("POD_IP environment variable not set")
|
||||
}
|
||||
|
||||
seedProvider := []map[string]interface{}{
|
||||
{
|
||||
"class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
|
||||
"parameters": []map[string]interface{}{
|
||||
{
|
||||
"seeds": seeds,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
config["cluster_name"] = m.cluster
|
||||
config["listen_address"] = localIP
|
||||
config["broadcast_address"] = m.ip
|
||||
config["rpc_address"] = "0.0.0.0"
|
||||
config["broadcast_rpc_address"] = m.ip
|
||||
config["endpoint_snitch"] = "GossipingPropertyFileSnitch"
|
||||
config["seed_provider"] = seedProvider
|
||||
|
||||
return yaml.Marshal(config)
|
||||
}
|
||||
|
||||
// getSeeds gets the IPs of the instances acting as Seeds
|
||||
// in the Cluster. It does that by getting all ClusterIP services
|
||||
// of the current Cluster with the cassandra.rook.io/seed label
|
||||
func (m *MemberController) getSeeds() (string, error) {
|
||||
|
||||
var services *corev1.ServiceList
|
||||
var err error
|
||||
|
||||
m.logger.Infof("Attempting to find seeds.")
|
||||
sel := fmt.Sprintf("%s,%s=%s", constants.SeedLabel, constants.ClusterNameLabel, m.cluster)
|
||||
|
||||
for {
|
||||
|
||||
services, err = m.kubeClient.CoreV1().Services(m.namespace).List(metav1.ListOptions{LabelSelector: sel})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(services.Items) > 0 {
|
||||
break
|
||||
}
|
||||
time.Sleep(1000 * time.Millisecond)
|
||||
}
|
||||
|
||||
seeds := []string{}
|
||||
for _, svc := range services.Items {
|
||||
seeds = append(seeds, svc.Spec.ClusterIP)
|
||||
}
|
||||
return strings.Join(seeds, ","), nil
|
||||
}
|
||||
|
||||
func getJolokiaConfig() string {
|
||||
|
||||
opts := []struct {
|
||||
flag, value string
|
||||
}{
|
||||
{
|
||||
flag: "host",
|
||||
value: "localhost",
|
||||
},
|
||||
{
|
||||
flag: "port",
|
||||
value: fmt.Sprintf("%d", constants.JolokiaPort),
|
||||
},
|
||||
{
|
||||
flag: "executor",
|
||||
value: "fixed",
|
||||
},
|
||||
{
|
||||
flag: "threadNr",
|
||||
value: "2",
|
||||
},
|
||||
}
|
||||
|
||||
cmd := []string{}
|
||||
for _, opt := range opts {
|
||||
cmd = append(cmd, fmt.Sprintf("%s=%s", opt.flag, opt.value))
|
||||
}
|
||||
return fmt.Sprintf("-javaagent:%s=%s", jolokiaPath, strings.Join(cmd, ","))
|
||||
}
|
||||
|
||||
// Merge YAMLs merges two arbitrary YAML structures on the top level.
|
||||
func mergeYAMLs(initialYAML, overrideYAML []byte) ([]byte, error) {
|
||||
|
||||
var initial, override map[string]interface{}
|
||||
yaml.Unmarshal(initialYAML, &initial)
|
||||
yaml.Unmarshal(overrideYAML, &override)
|
||||
|
||||
if initial == nil {
|
||||
initial = make(map[string]interface{})
|
||||
}
|
||||
// Overwrite the values onto initial
|
||||
for k, v := range override {
|
||||
initial[k] = v
|
||||
}
|
||||
return yaml.Marshal(initial)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package sidecar
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1"
|
||||
rookClientset "github.com/rook/rook/pkg/client/clientset/versioned"
|
||||
"github.com/rook/rook/pkg/operator/cassandra/constants"
|
||||
"github.com/yanniszark/go-nodetool/nodetool"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemberController encapsulates all the tools the sidecar needs to
|
||||
// talk to the Kubernetes API
|
||||
type MemberController struct {
|
||||
// Metadata of the specific Member
|
||||
name, namespace, ip string
|
||||
cluster, datacenter, rack string
|
||||
mode cassandrav1alpha1.ClusterMode
|
||||
|
||||
// Clients to handle Kubernetes Objects
|
||||
kubeClient kubernetes.Interface
|
||||
rookClient rookClientset.Interface
|
||||
|
||||
nodetool *nodetool.Nodetool
|
||||
queue workqueue.RateLimitingInterface
|
||||
logger *capnslog.PackageLogger
|
||||
}
|
||||
|
||||
// New return a new MemberController
|
||||
func New(
|
||||
name, namespace string,
|
||||
kubeClient kubernetes.Interface,
|
||||
rookClient rookClientset.Interface,
|
||||
) (*MemberController, error) {
|
||||
|
||||
logger := capnslog.NewPackageLogger("github.com/rook/rook", "sidecar")
|
||||
|
||||
// Get the member's service
|
||||
var memberService *corev1.Service
|
||||
var err error
|
||||
for {
|
||||
memberService, err = kubeClient.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Infof("Something went wrong trying to get Member Service %s", name)
|
||||
|
||||
} else if len(memberService.Spec.ClusterIP) > 0 {
|
||||
break
|
||||
}
|
||||
// If something went wrong, wait a little and retry
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Get the Member's metadata from the Pod's labels
|
||||
pod, err := kubeClient.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create a new nodetool interface to talk to Cassandra
|
||||
url, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d/jolokia/", constants.JolokiaPort))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodetool := nodetool.NewFromURL(url)
|
||||
|
||||
// Get the member's cluster
|
||||
cluster, err := rookClient.CassandraV1alpha1().Clusters(namespace).Get(pod.Labels[constants.ClusterNameLabel], metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m := &MemberController{
|
||||
name: name,
|
||||
namespace: namespace,
|
||||
ip: memberService.Spec.ClusterIP,
|
||||
cluster: pod.Labels[constants.ClusterNameLabel],
|
||||
datacenter: pod.Labels[constants.DatacenterNameLabel],
|
||||
rack: pod.Labels[constants.RackNameLabel],
|
||||
mode: cluster.Spec.Mode,
|
||||
kubeClient: kubeClient,
|
||||
rookClient: rookClient,
|
||||
nodetool: nodetool,
|
||||
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Run starts executing the sync loop for the sidecar
|
||||
func (m *MemberController) Run(threadiness int, stopCh <-chan struct{}) error {
|
||||
|
||||
defer runtime.HandleCrash()
|
||||
|
||||
if err := m.onStartup(); err != nil {
|
||||
return fmt.Errorf("error on startup: %s", err.Error())
|
||||
}
|
||||
|
||||
<-stopCh
|
||||
m.logger.Info("Shutting down sidecar.")
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// onStartup is executed before the MemberController starts
|
||||
// its sync loop.
|
||||
func (m *MemberController) onStartup() error {
|
||||
|
||||
// Setup HTTP checks
|
||||
m.logger.Info("Setting up HTTP Checks...")
|
||||
go func() {
|
||||
err := m.setupHTTPChecks()
|
||||
m.logger.Fatalf("Error with HTTP Server: %s", err.Error())
|
||||
panic("Something went wrong with the HTTP Checks")
|
||||
}()
|
||||
|
||||
// Prepare config files for Cassandra
|
||||
m.logger.Infof("Generating cassandra config files...")
|
||||
if err := m.generateConfigFiles(); err != nil {
|
||||
return fmt.Errorf("error generating config files: %s", err.Error())
|
||||
}
|
||||
|
||||
// Start the database daemon
|
||||
cmd := exec.Command(entrypointPath)
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Env = os.Environ()
|
||||
if err := cmd.Start(); err != nil {
|
||||
m.logger.Errorf("error starting database daemon: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user