helm
/
helm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement before-hook-creation delete policy 

Existing helm.sh/hook-delete-policy annotation variables (hook-failed, hook-succeeded) do not allow to leave failed jobs for debugging without blocking the next job launching: every failed job must be deleted manually before the next related release is launching (installing, updating or rolling back).

New policy, before-hook-creation, removes the hook from previous release if there is one before the new hook is launched and can be used with another variable.
This commit is contained in:
Alexey igrychev 2018-03-21 17:19:53 +03:00
parent 48e7039970
commit 1d4883bf3c
6 changed files with 97 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
_proto/hapi/release/hook.proto
View File

@ -38,6 +38,7 @@ message Hook {
enum DeletePolicy {
SUCCEEDED = 0;
FAILED = 1;
BEFORE_HOOK_CREATION = 2;
}
string name = 1;
// Kind is the Kubernetes kind.

17
docs/charts_hooks.md
View File

@ -180,4 +180,19 @@ It is also possible to define policies that determine when to delete correspondi
"helm.sh/hook-delete-policy": hook-succeeded
```
When using `"helm.sh/hook-delete-policy"` annotation, you can choose its value from `"hook-succeeded"` and `"hook-failed"`. The value `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed, while the value `"hook-failed"`specifies Tiller should delete the hook if the hook failed during execution.
You can choose one or more defined annotation values:
* `"hook-succeeded"` specifies Tiller should delete the hook after the hook is successfully executed.
* `"hook-failed"` specifies Tiller should delete the hook if the hook failed during execution.
* `"before-hook-creation"` specifies Tiller should delete the previous hook before the new hook is launched.
### Automatically delete hook from previous release
When helm release being updated it is possible, that hook resource already exists in cluster. By default helm will try to create resource and fail with `"... already exists"` error.
One might choose `"helm.sh/hook-delete-policy": "before-hook-creation"` over `"helm.sh/hook-delete-policy": "hook-succeeded,hook-failed"` because:
* It is convinient to keep failed hook job resource in kubernetes for example for manual debug.
* It may be necessary to keep succeeded hook resource in kubernetes for some reason.
* At the same time it is not desireable to do manual resource deletion before helm release upgrade.
`"helm.sh/hook-delete-policy": "before-hook-creation"` annotation on hook causes tiller to remove the hook from previous release if there is one before the new hook is launched and can be used with another policies.

5
pkg/hooks/hooks.go
View File

@ -45,8 +45,9 @@ const (
// Type of policy for deleting the hook
const (
HookSucceeded = "hook-succeeded"
HookFailed = "hook-failed"
HookSucceeded = "hook-succeeded"
HookFailed = "hook-failed"
BeforeHookCreation = "before-hook-creation"
)
// FilterTestHooks filters the list of hooks are returns only testing hooks.

68
pkg/proto/hapi/release/hook.pb.go
View File

@ -89,17 +89,20 @@ func (Hook_Event) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []i
type Hook_DeletePolicy int32
const (
Hook_SUCCEEDED Hook_DeletePolicy = 0
Hook_FAILED Hook_DeletePolicy = 1
Hook_SUCCEEDED Hook_DeletePolicy = 0
Hook_FAILED Hook_DeletePolicy = 1
Hook_BEFORE_HOOK_CREATION Hook_DeletePolicy = 2
)
var Hook_DeletePolicy_name = map[int32]string{
0: "SUCCEEDED",
1: "FAILED",
2: "BEFORE_HOOK_CREATION",
}
var Hook_DeletePolicy_value = map[string]int32{
"SUCCEEDED": 0,
"FAILED": 1,
"SUCCEEDED": 0,
"FAILED": 1,
"BEFORE_HOOK_CREATION": 2,
}
func (x Hook_DeletePolicy) String() string {
@ -196,32 +199,33 @@ func init() {
func init() { proto.RegisterFile("hapi/release/hook.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 428 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xd1, 0x6e, 0xda, 0x30,
0x14, 0x86, 0x9b, 0x12, 0x12, 0x38, 0x50, 0xea, 0x59, 0xd3, 0x66, 0x71, 0x53, 0xc4, 0x15, 0xbb,
0x09, 0x53, 0xa7, 0x3d, 0x40, 0x4a, 0xce, 0xd6, 0xaa, 0x11, 0x20, 0x27, 0x68, 0xd2, 0x6e, 0xa2,
0x74, 0xb8, 0x10, 0x11, 0xe2, 0x88, 0x98, 0x4d, 0x7b, 0xa6, 0xbd, 0xce, 0x1e, 0x68, 0xb2, 0x09,
0x59, 0xa5, 0xed, 0xee, 0x9c, 0xef, 0x7c, 0x76, 0xce, 0x1f, 0xc3, 0xdb, 0x6d, 0x5a, 0x66, 0xd3,
0x83, 0xc8, 0x45, 0x5a, 0x89, 0xe9, 0x56, 0xca, 0x9d, 0x57, 0x1e, 0xa4, 0x92, 0xb4, 0xaf, 0x07,
0x5e, 0x3d, 0x18, 0xde, 0x6c, 0xa4, 0xdc, 0xe4, 0x62, 0x6a, 0x66, 0x4f, 0xc7, 0xe7, 0xa9, 0xca,
0xf6, 0xa2, 0x52, 0xe9, 0xbe, 0x3c, 0xe9, 0xe3, 0x5f, 0x36, 0xd8, 0xf7, 0x52, 0xee, 0x28, 0x05,
0xbb, 0x48, 0xf7, 0x82, 0x59, 0x23, 0x6b, 0xd2, 0xe5, 0xa6, 0xd6, 0x6c, 0x97, 0x15, 0x6b, 0x76,
0x79, 0x62, 0xba, 0xd6, 0xac, 0x4c, 0xd5, 0x96, 0xb5, 0x4e, 0x4c, 0xd7, 0x74, 0x08, 0x9d, 0x7d,
0x5a, 0x64, 0xcf, 0xa2, 0x52, 0xcc, 0x36, 0xbc, 0xe9, 0xe9, 0x7b, 0x70, 0xc4, 0x77, 0x51, 0xa8,
0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x96, 0x79, 0x2f, 0x17, 0xf4, 0xf4, 0xb7, 0x3d, 0xd4, 0x02,
0xaf, 0x3d, 0xfa, 0x11, 0x3a, 0x79, 0x5a, 0xa9, 0xe4, 0x70, 0x2c, 0x98, 0x33, 0xb2, 0x26, 0xbd,
0xdb, 0xa1, 0x77, 0x8a, 0xe1, 0x9d, 0x63, 0x78, 0xf1, 0x39, 0x06, 0x77, 0xb5, 0xcb, 0x8f, 0x05,
0x7d, 0x03, 0xce, 0x0f, 0x91, 0x6d, 0xb6, 0x8a, 0xb9, 0x23, 0x6b, 0xd2, 0xe6, 0x75, 0x47, 0xef,
0xe1, 0x7a, 0x2d, 0x72, 0xa1, 0x44, 0x52, 0xca, 0x3c, 0xfb, 0x96, 0x89, 0x8a, 0x75, 0xcc, 0x26,
0x37, 0xff, 0xd9, 0x24, 0x30, 0xe6, 0x52, 0x8b, 0x3f, 0xf9, 0x60, 0xfd, 0xb7, 0xcb, 0x44, 0x35,
0xfe, 0x6d, 0x41, 0xdb, 0xac, 0x4a, 0x7b, 0xe0, 0xae, 0xe6, 0x8f, 0xf3, 0xc5, 0x97, 0x39, 0xb9,
0xa0, 0xd7, 0xd0, 0x5b, 0x72, 0x4c, 0x1e, 0xe6, 0x51, 0xec, 0x87, 0x21, 0xb1, 0x28, 0x81, 0xfe,
0x72, 0x11, 0xc5, 0x0d, 0xb9, 0xa4, 0x03, 0x00, 0xad, 0x04, 0x18, 0x62, 0x8c, 0xa4, 0x65, 0x8e,
0x68, 0xa3, 0x06, 0xf6, 0xf9, 0x8e, 0xd5, 0xf2, 0x33, 0xf7, 0x03, 0x24, 0xed, 0xe6, 0x8e, 0x33,
0x71, 0x0c, 0xe1, 0x98, 0xf0, 0x45, 0x18, 0xde, 0xf9, 0xb3, 0x47, 0xe2, 0xd2, 0x57, 0x70, 0x65,
0x9c, 0x06, 0x75, 0x28, 0x83, 0xd7, 0x1c, 0x43, 0xf4, 0x23, 0x4c, 0x62, 0x8c, 0xe2, 0x24, 0x5a,
0xcd, 0x66, 0x18, 0x45, 0xa4, 0xfb, 0xcf, 0xe4, 0x93, 0xff, 0x10, 0xae, 0x38, 0x12, 0x18, 0xbf,
0x83, 0xfe, 0xcb, 0xd8, 0xf4, 0x0a, 0xba, 0xe6, 0x18, 0x06, 0x18, 0x90, 0x0b, 0x0a, 0xe0, 0x68,
0x17, 0x03, 0x62, 0xdd, 0x75, 0xbf, 0xba, 0xf5, 0xef, 0x7a, 0x72, 0xcc, 0x5b, 0x7c, 0xf8, 0x13,
0x00, 0x00, 0xff, 0xff, 0xb9, 0x8a, 0xe1, 0xaf, 0x89, 0x02, 0x00, 0x00,
// 445 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x51, 0x8f, 0x9a, 0x40,
0x10, 0x80, 0x8f, 0x13, 0x41, 0x47, 0xcf, 0xdb, 0x6e, 0x9a, 0x76, 0xe3, 0xcb, 0x19, 0x9f, 0x7c,
0xc2, 0xe6, 0x9a, 0xfe, 0x00, 0x84, 0xb9, 0x6a, 0x24, 0x60, 0x16, 0x4c, 0x93, 0xbe, 0x10, 0xae,
0xee, 0x29, 0x11, 0x81, 0x08, 0xb6, 0xe9, 0x0f, 0xec, 0x3f, 0xe8, 0x0f, 0x6a, 0x76, 0x45, 0x7b,
0x49, 0xfb, 0x36, 0xf3, 0xcd, 0x37, 0xc3, 0x0c, 0x0b, 0xef, 0x77, 0x49, 0x99, 0x4e, 0x8f, 0x22,
0x13, 0x49, 0x25, 0xa6, 0xbb, 0xa2, 0xd8, 0x5b, 0xe5, 0xb1, 0xa8, 0x0b, 0xda, 0x97, 0x05, 0xab,
0x29, 0x0c, 0x1f, 0xb6, 0x45, 0xb1, 0xcd, 0xc4, 0x54, 0xd5, 0x9e, 0x4f, 0x2f, 0xd3, 0x3a, 0x3d,
0x88, 0xaa, 0x4e, 0x0e, 0xe5, 0x59, 0x1f, 0xff, 0xd2, 0x41, 0x9f, 0x17, 0xc5, 0x9e, 0x52, 0xd0,
0xf3, 0xe4, 0x20, 0x98, 0x36, 0xd2, 0x26, 0x5d, 0xae, 0x62, 0xc9, 0xf6, 0x69, 0xbe, 0x61, 0xb7,
0x67, 0x26, 0x63, 0xc9, 0xca, 0xa4, 0xde, 0xb1, 0xd6, 0x99, 0xc9, 0x98, 0x0e, 0xa1, 0x73, 0x48,
0xf2, 0xf4, 0x45, 0x54, 0x35, 0xd3, 0x15, 0xbf, 0xe6, 0xf4, 0x03, 0x18, 0xe2, 0xbb, 0xc8, 0xeb,
0x8a, 0xb5, 0x47, 0xad, 0xc9, 0xe0, 0x91, 0x59, 0xaf, 0x17, 0xb4, 0xe4, 0xb7, 0x2d, 0x94, 0x02,
0x6f, 0x3c, 0xfa, 0x09, 0x3a, 0x59, 0x52, 0xd5, 0xf1, 0xf1, 0x94, 0x33, 0x63, 0xa4, 0x4d, 0x7a,
0x8f, 0x43, 0xeb, 0x7c, 0x86, 0x75, 0x39, 0xc3, 0x8a, 0x2e, 0x67, 0x70, 0x53, 0xba, 0xfc, 0x94,
0xd3, 0x77, 0x60, 0xfc, 0x10, 0xe9, 0x76, 0x57, 0x33, 0x73, 0xa4, 0x4d, 0xda, 0xbc, 0xc9, 0xe8,
0x1c, 0xee, 0x37, 0x22, 0x13, 0xb5, 0x88, 0xcb, 0x22, 0x4b, 0xbf, 0xa5, 0xa2, 0x62, 0x1d, 0xb5,
0xc9, 0xc3, 0x7f, 0x36, 0x71, 0x95, 0xb9, 0x92, 0xe2, 0x4f, 0x3e, 0xd8, 0xfc, 0xcd, 0x52, 0x51,
0x8d, 0x7f, 0x6b, 0xd0, 0x56, 0xab, 0xd2, 0x1e, 0x98, 0x6b, 0x7f, 0xe9, 0x07, 0x5f, 0x7c, 0x72,
0x43, 0xef, 0xa1, 0xb7, 0xe2, 0x18, 0x2f, 0xfc, 0x30, 0xb2, 0x3d, 0x8f, 0x68, 0x94, 0x40, 0x7f,
0x15, 0x84, 0xd1, 0x95, 0xdc, 0xd2, 0x01, 0x80, 0x54, 0x5c, 0xf4, 0x30, 0x42, 0xd2, 0x52, 0x2d,
0xd2, 0x68, 0x80, 0x7e, 0x99, 0xb1, 0x5e, 0x7d, 0xe6, 0xb6, 0x8b, 0xa4, 0x7d, 0x9d, 0x71, 0x21,
0x86, 0x22, 0x1c, 0x63, 0x1e, 0x78, 0xde, 0xcc, 0x76, 0x96, 0xc4, 0xa4, 0x6f, 0xe0, 0x4e, 0x39,
0x57, 0xd4, 0xa1, 0x0c, 0xde, 0x72, 0xf4, 0xd0, 0x0e, 0x31, 0x8e, 0x30, 0x8c, 0xe2, 0x70, 0xed,
0x38, 0x18, 0x86, 0xa4, 0xfb, 0x4f, 0xe5, 0xc9, 0x5e, 0x78, 0x6b, 0x8e, 0x04, 0xc6, 0x0e, 0xf4,
0x5f, 0x9f, 0x4d, 0xef, 0xa0, 0xab, 0xda, 0xd0, 0x45, 0x97, 0xdc, 0x50, 0x00, 0x43, 0xba, 0xe8,
0x12, 0x4d, 0x0e, 0x99, 0xe1, 0x53, 0xc0, 0x31, 0x9e, 0x07, 0xc1, 0x32, 0x76, 0x38, 0xda, 0xd1,
0x22, 0xf0, 0xc9, 0xed, 0xac, 0xfb, 0xd5, 0x6c, 0x7e, 0xe4, 0xb3, 0xa1, 0x5e, 0xe9, 0xe3, 0x9f,
0x00, 0x00, 0x00, 0xff, 0xff, 0x13, 0x64, 0x75, 0x6c, 0xa3, 0x02, 0x00, 0x00,
}

35
pkg/tiller/hooks.go
View File

@ -46,8 +46,9 @@ var events = map[string]release.Hook_Event{
// deletePolices represents a mapping between the key in the annotation for label deleting policy and its real meaning
var deletePolices = map[string]release.Hook_DeletePolicy{
hooks.HookSucceeded: release.Hook_SUCCEEDED,
hooks.HookFailed: release.Hook_FAILED,
hooks.HookSucceeded: release.Hook_SUCCEEDED,
hooks.HookFailed: release.Hook_FAILED,
hooks.BeforeHookCreation: release.Hook_BEFORE_HOOK_CREATION,
}
// Manifest represents a manifest file, which has a name and some content.
@ -189,21 +190,14 @@ func (file *manifestFile) sort(result *result) error {
result.hooks = append(result.hooks, h)
isKnownDeletePolices := false
dps, ok := entry.Metadata.Annotations[hooks.HookDeleteAnno]
if ok {
for _, dp := range strings.Split(dps, ",") {
dp = strings.ToLower(strings.TrimSpace(dp))
p, exist := deletePolices[dp]
if exist {
isKnownDeletePolices = true
h.DeletePolicies = append(h.DeletePolicies, p)
}
operateAnnotationValues(entry, hooks.HookDeleteAnno, func(value string) {
policy, exist := deletePolices[value]
if exist {
h.DeletePolicies = append(h.DeletePolicies, policy)
} else {
log.Printf("info: skipping unknown hook delete policy: %q", value)
}
if !isKnownDeletePolices {
log.Printf("info: skipping unknown hook delete policy: %q", dps)
}
}
})
}
return nil
@ -228,3 +222,12 @@ func calculateHookWeight(entry util.SimpleHead) int32 {
return int32(hw)
}
func operateAnnotationValues(entry util.SimpleHead, annotation string, operate func(p string)) {
if dps, ok := entry.Metadata.Annotations[annotation]; ok {
for _, dp := range strings.Split(dps, ",") {
dp = strings.ToLower(strings.TrimSpace(dp))
operate(dp)
}
}
}

39
pkg/tiller/release_server.go
View File

@ -347,6 +347,9 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
executingHooks = sortByHookWeight(executingHooks)
for _, h := range executingHooks {
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.BeforeHookCreation, name, namespace, hook, kubeCli); err != nil {
return err
}
b := bytes.NewBufferString(h.Manifest)
if err := kubeCli.Create(namespace, b, timeout, false); err != nil {
@ -356,18 +359,13 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
// No way to rewind a bytes.Buffer()?
b.Reset()
b.WriteString(h.Manifest)
if err := kubeCli.WatchUntilReady(namespace, b, timeout, false); err != nil {
s.Log("warning: Release %s %s %s could not complete: %s", name, hook, h.Path, err)
// If a hook is failed, checkout the annotation of the hook to determine whether the hook should be deleted
// under failed condition. If so, then clear the corresponding resource object in the hook
if hookShouldBeDeleted(h, hooks.HookFailed) {
b.Reset()
b.WriteString(h.Manifest)
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookFailed)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookFailed, name, namespace, hook, kubeCli); err != nil {
return err
}
return err
}
@ -377,13 +375,8 @@ func (s *ReleaseServer) execHook(hs []*release.Hook, name, namespace, hook strin
// If all hooks are succeeded, checkout the annotation of each hook to determine whether the hook should be deleted
// under succeeded condition. If so, then clear the corresponding resource object in each hook
for _, h := range executingHooks {
b := bytes.NewBufferString(h.Manifest)
if hookShouldBeDeleted(h, hooks.HookSucceeded) {
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, hooks.HookSucceeded)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
if err := s.deleteHookIfShouldBeDeletedByDeletePolicy(h, hooks.HookSucceeded, name, namespace, hook, kubeCli); err != nil {
return err
}
h.LastRun = timeconv.Now()
}
@ -409,11 +402,23 @@ func validateReleaseName(releaseName string) error {
return nil
}
func (s *ReleaseServer) deleteHookIfShouldBeDeletedByDeletePolicy(h *release.Hook, policy string, name, namespace, hook string, kubeCli environment.KubeClient) error {
b := bytes.NewBufferString(h.Manifest)
if hookHasDeletePolicy(h, policy) {
s.Log("deleting %s hook %s for release %s due to %q policy", hook, h.Name, name, policy)
if errHookDelete := kubeCli.Delete(namespace, b); errHookDelete != nil {
s.Log("warning: Release %s %s %S could not be deleted: %s", name, hook, h.Path, errHookDelete)
return errHookDelete
}
}
return nil
}
// hookShouldBeDeleted determines whether the defined hook deletion policy matches the hook deletion polices
// supported by helm. If so, mark the hook as one should be deleted.
func hookShouldBeDeleted(hook *release.Hook, policy string) bool {
func hookHasDeletePolicy(h *release.Hook, policy string) bool {
if dp, ok := deletePolices[policy]; ok {
for _, v := range hook.DeletePolicies {
for _, v := range h.DeletePolicies {
if dp == v {
return true
}