forked from rook/rook
Satoru Takeuchi
a89d35e5b6
Sometimes `radosgw-admin` succeeds after showing logs to stderr. We should
skip non-json strings if parsing output as json.
Here is an example.
```
2021-02-26 04:10:44.190418 I | op-bucket-prov: creating Ceph user "ceph-user-aSzNqgE7"
E0226 04:11:37.901310 8 controller.go:199] error syncing 'logging/loki-bucket': error provisioning bucket: Provision: can't create ceph user: error creating ceph user "ceph-user-aSzNqgE7": failed to unmarshal json. 2021-02-26T04:11:21.425+0000 7f6714be4980 1 robust_notify: If at first you don't succeed: (110) Connection timed out
2021-02-26T04:11:21.426+0000 7f6714be4980 0 ERROR: failed to distribute cache for ceph-hdd-object-store.rgw.meta:users.uid:ceph-user-aSzNqgE7
2021-02-26T04:11:32.168+0000 7f6714be4980 1 robust_notify: If at first you don't succeed: (110) Connection timed out
2021-02-26T04:11:32.168+0000 7f6714be4980 0 ERROR: failed to distribute cache for ceph-hdd-object-store.rgw.meta:users.keys:23Z8GUEXR0TJDO86PSJR
{
"user_id": "ceph-user-aSzNqgE7",
"display_name": "ceph-user-aSzNqgE7",
...
"mfa_ids": []
}: invalid character '-' after top-level value: failed to unmarshal json.
```
In this case, some logs like "robust_notify:..." was shown in stderr.
Unmarsharing was failed due to tried to parse these logs as json.
Signed-off-by: Satoru Takeuchi <satoru.takeuchi@gmail.com>
363 lines
10 KiB
Go
363 lines
10 KiB
Go
/*
|
|
Copyright 2020 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
// Package zone to manage a rook object zone.
|
|
package zone
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
|
"github.com/rook/rook/pkg/operator/test"
|
|
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
)
|
|
|
|
const (
|
|
zoneGroupGetJSON = `{
|
|
"id": "fd8ff110-d3fd-49b4-b24f-f6cd3dddfedf",
|
|
"name": "zonegroup-a",
|
|
"api_name": "zonegroup-a",
|
|
"is_master": "true",
|
|
"endpoints": [
|
|
":80"
|
|
],
|
|
"hostnames": [],
|
|
"hostnames_s3website": [],
|
|
"master_zone": "6cb39d2c-3005-49da-9be3-c1a92a97d28a",
|
|
"zones": [
|
|
{
|
|
"id": "6cb39d2c-3005-49da-9be3-c1a92a97d28a",
|
|
"name": "zone-group",
|
|
"endpoints": [
|
|
":80"
|
|
],
|
|
"log_meta": "false",
|
|
"log_data": "false",
|
|
"bucket_index_max_shards": 0,
|
|
"read_only": "false",
|
|
"tier_type": "",
|
|
"sync_from_all": "true",
|
|
"sync_from": [],
|
|
"redirect_zone": ""
|
|
}
|
|
],
|
|
"placement_targets": [
|
|
{
|
|
"name": "default-placement",
|
|
"tags": [],
|
|
"storage_classes": [
|
|
"STANDARD"
|
|
]
|
|
}
|
|
],
|
|
"default_placement": "default-placement",
|
|
"realm_id": "237e6250-5f7d-4b85-9359-8cb2b1848507"
|
|
}`
|
|
zoneGetOutput = `{"id": "test-id"}`
|
|
zoneCreateJSON = `{
|
|
"id": "b1abbebb-e8ae-4c3b-880e-b009728bad53",
|
|
"name": "zone-a",
|
|
"domain_root": "zone-a.rgw.meta:root",
|
|
"control_pool": "zone-a.rgw.control",
|
|
"gc_pool": "zone-a.rgw.log:gc",
|
|
"lc_pool": "zone-a.rgw.log:lc",
|
|
"log_pool": "zone-a.rgw.log",
|
|
"intent_log_pool": "zone-a.rgw.log:intent",
|
|
"usage_log_pool": "zone-a.rgw.log:usage",
|
|
"roles_pool": "zone-a.rgw.meta:roles",
|
|
"reshard_pool": "zone-a.rgw.log:reshard",
|
|
"user_keys_pool": "zone-a.rgw.meta:users.keys",
|
|
"user_email_pool": "zone-a.rgw.meta:users.email",
|
|
"user_swift_pool": "zone-a.rgw.meta:users.swift",
|
|
"user_uid_pool": "zone-a.rgw.meta:users.uid",
|
|
"otp_pool": "zone-a.rgw.otp",
|
|
"system_key": {
|
|
"access_key": "",
|
|
"secret_key": ""
|
|
},
|
|
"placement_pools": [
|
|
{
|
|
"key": "default-placement",
|
|
"val": {
|
|
"index_pool": "zone-a.rgw.buckets.index",
|
|
"storage_classes": {
|
|
"STANDARD": {
|
|
"data_pool": "zone-a.rgw.buckets.data"
|
|
}
|
|
},
|
|
"data_extra_pool": "zone-a.rgw.buckets.non-ec",
|
|
"index_type": 0
|
|
}
|
|
}
|
|
],
|
|
"realm_id": "91b799b2-857d-4c96-8ade-5ceff7c8597e"
|
|
}`
|
|
)
|
|
|
|
func TestCephObjectZoneController(t *testing.T) {
|
|
ctx := context.TODO()
|
|
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
|
name := "zone-a"
|
|
zonegroup := "zonegroup-a"
|
|
namespace := "rook-ceph"
|
|
|
|
//
|
|
// TEST 1 SETUP
|
|
//
|
|
// FAILURE: because no CephCluster
|
|
//
|
|
// A Pool resource with metadata and spec.
|
|
metadataPool := cephv1.PoolSpec{}
|
|
dataPool := cephv1.PoolSpec{}
|
|
objectZone := &cephv1.CephObjectZone{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephObjectZone",
|
|
},
|
|
Spec: cephv1.ObjectZoneSpec{
|
|
ZoneGroup: zonegroup,
|
|
MetadataPool: metadataPool,
|
|
DataPool: dataPool,
|
|
},
|
|
}
|
|
|
|
// Objects to track in the fake client.
|
|
object := []runtime.Object{
|
|
objectZone,
|
|
}
|
|
|
|
executor := &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutputFile: func(command, outfile string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
clientset := test.New(t, 3)
|
|
c := &clusterd.Context{
|
|
Executor: executor,
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
Clientset: clientset,
|
|
}
|
|
|
|
// Register operator types with the runtime scheme.
|
|
s := scheme.Scheme
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{})
|
|
|
|
// Create a fake client to mock API calls.
|
|
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
// Create a ReconcileObjectZone object with the scheme and fake client.
|
|
clusterInfo := cephclient.AdminClusterInfo("rook")
|
|
|
|
r := &ReconcileObjectZone{client: cl, scheme: s, context: c, clusterInfo: clusterInfo}
|
|
|
|
// Mock request to simulate Reconcile() being called on an event for a
|
|
// watched resource .
|
|
req := reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
}
|
|
|
|
res, err := r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
|
|
//
|
|
// TEST 2:
|
|
//
|
|
// FAILURE: we have a cluster but it's not ready
|
|
//
|
|
cephCluster := &cephv1.CephCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Status: cephv1.ClusterStatus{
|
|
Phase: "",
|
|
CephStatus: &cephv1.CephStatus{
|
|
Health: "",
|
|
},
|
|
},
|
|
}
|
|
|
|
object = []runtime.Object{
|
|
objectZone,
|
|
cephCluster,
|
|
}
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
|
|
cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
// Create a ReconcileObjectZone object with the scheme and fake client.
|
|
r = &ReconcileObjectZone{client: cl, scheme: r.scheme, context: r.context}
|
|
res, err = r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.True(t, res.Requeue)
|
|
|
|
//
|
|
// TEST 3:
|
|
//
|
|
// Failure: The CephCluster is ready but no ObjectZoneGroup has been created
|
|
//
|
|
|
|
// Mock clusterInfo
|
|
secrets := map[string][]byte{
|
|
"fsid": []byte(name),
|
|
"mon-secret": []byte("monsecret"),
|
|
"admin-secret": []byte("adminsecret"),
|
|
}
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rook-ceph-mon",
|
|
Namespace: namespace,
|
|
},
|
|
Data: secrets,
|
|
Type: k8sutil.RookType,
|
|
}
|
|
_, err = r.context.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
// Add ready status to the CephCluster
|
|
cephCluster.Status.Phase = k8sutil.ReadyStatus
|
|
cephCluster.Status.CephStatus.Health = "HEALTH_OK"
|
|
|
|
// Create a fake client to mock API calls.
|
|
cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutputFile: func(command, outfile string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
if args[0] == "zonegroup" && args[1] == "get" {
|
|
return zoneGroupGetJSON, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
r.context.Executor = executor
|
|
|
|
r = &ReconcileObjectZone{client: cl, scheme: r.scheme, context: r.context}
|
|
|
|
res, err = r.Reconcile(ctx, req)
|
|
assert.Error(t, err)
|
|
assert.True(t, res.Requeue)
|
|
|
|
//
|
|
// TEST 4:
|
|
//
|
|
// Success: The CephCluster is ready and ObjectZone has been created
|
|
//
|
|
|
|
objectZoneGroup := &cephv1.CephObjectZoneGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: zonegroup,
|
|
Namespace: namespace,
|
|
},
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "CephObjectZoneGroup",
|
|
},
|
|
Spec: cephv1.ObjectZoneGroupSpec{},
|
|
}
|
|
|
|
// Objects to track in the fake client.
|
|
object = []runtime.Object{
|
|
objectZone,
|
|
objectZoneGroup,
|
|
cephCluster,
|
|
}
|
|
|
|
executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutputFile: func(command, outfile string, args ...string) (string, error) {
|
|
if args[0] == "status" {
|
|
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) {
|
|
if args[0] == "zonegroup" && args[1] == "get" {
|
|
return zoneGroupGetJSON, nil
|
|
}
|
|
if args[0] == "zone" && args[1] == "get" {
|
|
return zoneGetOutput, nil
|
|
}
|
|
if args[0] == "zone" && args[1] == "create" {
|
|
return zoneCreateJSON, nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
|
|
clientset = test.New(t, 3)
|
|
c = &clusterd.Context{
|
|
Executor: executor,
|
|
RookClientset: rookclient.NewSimpleClientset(),
|
|
Clientset: clientset,
|
|
}
|
|
|
|
_, err = c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
assert.NoError(t, err)
|
|
|
|
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZoneGroup{}, &cephv1.CephObjectZoneGroupList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{})
|
|
|
|
cl = fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
|
|
|
r = &ReconcileObjectZone{client: cl, scheme: s, context: c, clusterInfo: clusterInfo}
|
|
|
|
err = r.client.Get(context.TODO(), types.NamespacedName{Name: zonegroup, Namespace: namespace}, objectZoneGroup)
|
|
assert.NoError(t, err, objectZoneGroup)
|
|
|
|
req = reconcile.Request{
|
|
NamespacedName: types.NamespacedName{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
}
|
|
|
|
res, err = r.Reconcile(ctx, req)
|
|
assert.NoError(t, err)
|
|
assert.False(t, res.Requeue)
|
|
err = r.client.Get(context.TODO(), req.NamespacedName, objectZone)
|
|
assert.NoError(t, err)
|
|
}
|