From 986cdfa6ffa09b9a0f81cb3f47135904136bc319 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Thu, 18 May 2017 23:59:17 -0400 Subject: [PATCH 1/9] bump(golang.org/x/tools/container/intsets): 2382e3994d48b1d22acc2c86bcad0a2aff028e32 --- Godeps/Godeps.json | 4 + Godeps/LICENSES | 35 + vendor/BUILD | 1 + vendor/golang.org/x/tools/AUTHORS | 3 + vendor/golang.org/x/tools/CONTRIBUTORS | 3 + vendor/golang.org/x/tools/LICENSE | 27 + vendor/golang.org/x/tools/PATENTS | 22 + .../x/tools/container/intsets/BUILD | 32 + .../x/tools/container/intsets/popcnt_amd64.go | 20 + .../x/tools/container/intsets/popcnt_amd64.s | 30 + .../x/tools/container/intsets/popcnt_gccgo.go | 9 + .../tools/container/intsets/popcnt_gccgo_c.c | 19 + .../tools/container/intsets/popcnt_generic.go | 33 + .../x/tools/container/intsets/sparse.go | 967 ++++++++++++++++++ .../x/tools/container/intsets/util.go | 84 ++ 15 files changed, 1289 insertions(+) create mode 100644 vendor/golang.org/x/tools/AUTHORS create mode 100644 vendor/golang.org/x/tools/CONTRIBUTORS create mode 100644 vendor/golang.org/x/tools/LICENSE create mode 100644 vendor/golang.org/x/tools/PATENTS create mode 100644 vendor/golang.org/x/tools/container/intsets/BUILD create mode 100644 vendor/golang.org/x/tools/container/intsets/popcnt_amd64.go create mode 100644 vendor/golang.org/x/tools/container/intsets/popcnt_amd64.s create mode 100644 vendor/golang.org/x/tools/container/intsets/popcnt_gccgo.go create mode 100644 vendor/golang.org/x/tools/container/intsets/popcnt_gccgo_c.c create mode 100644 vendor/golang.org/x/tools/container/intsets/popcnt_generic.go create mode 100644 vendor/golang.org/x/tools/container/intsets/sparse.go create mode 100644 vendor/golang.org/x/tools/container/intsets/util.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index cc4c30ff8ea..58ab4b3491c 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -2739,6 +2739,10 @@ "ImportPath": "golang.org/x/time/rate", "Rev": "f51c12702a4d776e4c1fa9b0fabab841babae631" }, + { + "ImportPath": "golang.org/x/tools/container/intsets", + "Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" + }, { "ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2", "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" diff --git a/Godeps/LICENSES b/Godeps/LICENSES index 26cba7a370c..57cb8e1b842 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -83172,6 +83172,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/golang.org/x/tools/container/intsets licensed under: = + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/golang.org/x/tools/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - +================================================================================ + + ================================================================================ = vendor/google.golang.org/api/cloudmonitoring/v2beta2 licensed under: = diff --git a/vendor/BUILD b/vendor/BUILD index 9f91330aa7b..89aeb8b66d7 100644 --- a/vendor/BUILD +++ b/vendor/BUILD @@ -346,6 +346,7 @@ filegroup( "//vendor/golang.org/x/text/unicode/norm:all-srcs", "//vendor/golang.org/x/text/width:all-srcs", "//vendor/golang.org/x/time/rate:all-srcs", + "//vendor/golang.org/x/tools/container/intsets:all-srcs", "//vendor/google.golang.org/api/cloudmonitoring/v2beta2:all-srcs", "//vendor/google.golang.org/api/compute/v0.alpha:all-srcs", "//vendor/google.golang.org/api/compute/v0.beta:all-srcs", diff --git a/vendor/golang.org/x/tools/AUTHORS b/vendor/golang.org/x/tools/AUTHORS new file mode 100644 index 00000000000..15167cd746c --- /dev/null +++ b/vendor/golang.org/x/tools/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/golang.org/x/tools/CONTRIBUTORS b/vendor/golang.org/x/tools/CONTRIBUTORS new file mode 100644 index 00000000000..1c4577e9680 --- /dev/null +++ b/vendor/golang.org/x/tools/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/golang.org/x/tools/LICENSE b/vendor/golang.org/x/tools/LICENSE new file mode 100644 index 00000000000..6a66aea5eaf --- /dev/null +++ b/vendor/golang.org/x/tools/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/tools/PATENTS b/vendor/golang.org/x/tools/PATENTS new file mode 100644 index 00000000000..733099041f8 --- /dev/null +++ b/vendor/golang.org/x/tools/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/tools/container/intsets/BUILD b/vendor/golang.org/x/tools/container/intsets/BUILD new file mode 100644 index 00000000000..c6b2fa906a0 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "popcnt_amd64.go", + "popcnt_amd64.s", + "sparse.go", + "util.go", + ], + tags = ["automanaged"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.go b/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.go new file mode 100644 index 00000000000..99ea813d284 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.go @@ -0,0 +1,20 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64,!appengine,!gccgo + +package intsets + +func popcnt(x word) int +func havePOPCNT() bool + +var hasPOPCNT = havePOPCNT() + +// popcount returns the population count (number of set bits) of x. +func popcount(x word) int { + if hasPOPCNT { + return popcnt(x) + } + return popcountTable(x) // faster than Hacker's Delight +} diff --git a/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.s b/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.s new file mode 100644 index 00000000000..05c3d6fb573 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/popcnt_amd64.s @@ -0,0 +1,30 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64,!appengine,!gccgo + +#include "textflag.h" + +// func havePOPCNT() bool +TEXT ·havePOPCNT(SB),4,$0 + MOVQ $1, AX + CPUID + SHRQ $23, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + +// func popcnt(word) int +TEXT ·popcnt(SB),NOSPLIT,$0-8 + XORQ AX, AX + MOVQ x+0(FP), SI + // POPCNT (SI), AX is not recognized by Go assembler, + // so we assemble it ourselves. + BYTE $0xf3 + BYTE $0x48 + BYTE $0x0f + BYTE $0xb8 + BYTE $0xc6 + MOVQ AX, ret+8(FP) + RET diff --git a/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo.go b/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo.go new file mode 100644 index 00000000000..82a8875c85d --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo.go @@ -0,0 +1,9 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build gccgo + +package intsets + +func popcount(x word) int diff --git a/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo_c.c b/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo_c.c new file mode 100644 index 00000000000..08abb32ec46 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/popcnt_gccgo_c.c @@ -0,0 +1,19 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build gccgo + +#include +#include +#include + +#define _STRINGIFY2_(x) #x +#define _STRINGIFY_(x) _STRINGIFY2_(x) +#define GOSYM_PREFIX _STRINGIFY_(__USER_LABEL_PREFIX__) + +extern intptr_t popcount(uintptr_t x) __asm__(GOSYM_PREFIX GOPKGPATH ".popcount"); + +intptr_t popcount(uintptr_t x) { + return __builtin_popcountl((unsigned long)(x)); +} diff --git a/vendor/golang.org/x/tools/container/intsets/popcnt_generic.go b/vendor/golang.org/x/tools/container/intsets/popcnt_generic.go new file mode 100644 index 00000000000..3985a1da1a2 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/popcnt_generic.go @@ -0,0 +1,33 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64 appengine +// +build !gccgo + +package intsets + +import "runtime" + +// We compared three algorithms---Hacker's Delight, table lookup, +// and AMD64's SSE4.1 hardware POPCNT---on a 2.67GHz Xeon X5550. +// +// % GOARCH=amd64 go test -run=NONE -bench=Popcount +// POPCNT 5.12 ns/op +// Table 8.53 ns/op +// HackersDelight 9.96 ns/op +// +// % GOARCH=386 go test -run=NONE -bench=Popcount +// Table 10.4 ns/op +// HackersDelight 5.23 ns/op +// +// (AMD64's ABM1 hardware supports ntz and nlz too, +// but they aren't critical.) + +// popcount returns the population count (number of set bits) of x. +func popcount(x word) int { + if runtime.GOARCH == "386" { + return popcountHD(uint32(x)) + } + return popcountTable(x) +} diff --git a/vendor/golang.org/x/tools/container/intsets/sparse.go b/vendor/golang.org/x/tools/container/intsets/sparse.go new file mode 100644 index 00000000000..adcdf40d5ab --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/sparse.go @@ -0,0 +1,967 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package intsets provides Sparse, a compact and fast representation +// for sparse sets of int values. +// +// The time complexity of the operations Len, Insert, Remove and Has +// is in O(n) but in practice those methods are faster and more +// space-efficient than equivalent operations on sets based on the Go +// map type. The IsEmpty, Min, Max, Clear and TakeMin operations +// require constant time. +// +package intsets + +// TODO(adonovan): +// - Add InsertAll(...int), RemoveAll(...int) +// - Add 'bool changed' results for {Intersection,Difference}With too. +// +// TODO(adonovan): implement Dense, a dense bit vector with a similar API. +// The space usage would be proportional to Max(), not Len(), and the +// implementation would be based upon big.Int. +// +// TODO(adonovan): experiment with making the root block indirect (nil +// iff IsEmpty). This would reduce the memory usage when empty and +// might simplify the aliasing invariants. +// +// TODO(adonovan): opt: make UnionWith and Difference faster. +// These are the hot-spots for go/pointer. + +import ( + "bytes" + "fmt" +) + +// A Sparse is a set of int values. +// Sparse operations (even queries) are not concurrency-safe. +// +// The zero value for Sparse is a valid empty set. +// +// Sparse sets must be copied using the Copy method, not by assigning +// a Sparse value. +// +type Sparse struct { + // An uninitialized Sparse represents an empty set. + // An empty set may also be represented by + // root.next == root.prev == &root. + // In a non-empty set, root.next points to the first block and + // root.prev to the last. + // root.offset and root.bits are unused. + root block +} + +type word uintptr + +const ( + _m = ^word(0) + bitsPerWord = 8 << (_m>>8&1 + _m>>16&1 + _m>>32&1) + bitsPerBlock = 256 // optimal value for go/pointer solver performance + wordsPerBlock = bitsPerBlock / bitsPerWord +) + +// Limit values of implementation-specific int type. +const ( + MaxInt = int(^uint(0) >> 1) + MinInt = -MaxInt - 1 +) + +// -- block ------------------------------------------------------------ + +// A set is represented as a circular doubly-linked list of blocks, +// each containing an offset and a bit array of fixed size +// bitsPerBlock; the blocks are ordered by increasing offset. +// +// The set contains an element x iff the block whose offset is x - (x +// mod bitsPerBlock) has the bit (x mod bitsPerBlock) set, where mod +// is the Euclidean remainder. +// +// A block may only be empty transiently. +// +type block struct { + offset int // offset mod bitsPerBlock == 0 + bits [wordsPerBlock]word // contains at least one set bit + next, prev *block // doubly-linked list of blocks +} + +// wordMask returns the word index (in block.bits) +// and single-bit mask for the block's ith bit. +func wordMask(i uint) (w uint, mask word) { + w = i / bitsPerWord + mask = 1 << (i % bitsPerWord) + return +} + +// insert sets the block b's ith bit and +// returns true if it was not already set. +// +func (b *block) insert(i uint) bool { + w, mask := wordMask(i) + if b.bits[w]&mask == 0 { + b.bits[w] |= mask + return true + } + return false +} + +// remove clears the block's ith bit and +// returns true if the bit was previously set. +// NB: may leave the block empty. +// +func (b *block) remove(i uint) bool { + w, mask := wordMask(i) + if b.bits[w]&mask != 0 { + b.bits[w] &^= mask + return true + } + return false +} + +// has reports whether the block's ith bit is set. +func (b *block) has(i uint) bool { + w, mask := wordMask(i) + return b.bits[w]&mask != 0 +} + +// empty reports whether b.len()==0, but more efficiently. +func (b *block) empty() bool { + for _, w := range b.bits { + if w != 0 { + return false + } + } + return true +} + +// len returns the number of set bits in block b. +func (b *block) len() int { + var l int + for _, w := range b.bits { + l += popcount(w) + } + return l +} + +// max returns the maximum element of the block. +// The block must not be empty. +// +func (b *block) max() int { + bi := b.offset + bitsPerBlock + // Decrement bi by number of high zeros in last.bits. + for i := len(b.bits) - 1; i >= 0; i-- { + if w := b.bits[i]; w != 0 { + return bi - nlz(w) - 1 + } + bi -= bitsPerWord + } + panic("BUG: empty block") +} + +// min returns the minimum element of the block, +// and also removes it if take is set. +// The block must not be initially empty. +// NB: may leave the block empty. +// +func (b *block) min(take bool) int { + for i, w := range b.bits { + if w != 0 { + tz := ntz(w) + if take { + b.bits[i] = w &^ (1 << uint(tz)) + } + return b.offset + int(i*bitsPerWord) + tz + } + } + panic("BUG: empty block") +} + +// forEach calls f for each element of block b. +// f must not mutate b's enclosing Sparse. +func (b *block) forEach(f func(int)) { + for i, w := range b.bits { + offset := b.offset + i*bitsPerWord + for bi := 0; w != 0 && bi < bitsPerWord; bi++ { + if w&1 != 0 { + f(offset) + } + offset++ + w >>= 1 + } + } +} + +// offsetAndBitIndex returns the offset of the block that would +// contain x and the bit index of x within that block. +// +func offsetAndBitIndex(x int) (int, uint) { + mod := x % bitsPerBlock + if mod < 0 { + // Euclidean (non-negative) remainder + mod += bitsPerBlock + } + return x - mod, uint(mod) +} + +// -- Sparse -------------------------------------------------------------- + +// start returns the root's next block, which is the root block +// (if s.IsEmpty()) or the first true block otherwise. +// start has the side effect of ensuring that s is properly +// initialized. +// +func (s *Sparse) start() *block { + root := &s.root + if root.next == nil { + root.next = root + root.prev = root + } else if root.next.prev != root { + // Copying a Sparse x leads to pernicious corruption: the + // new Sparse y shares the old linked list, but iteration + // on y will never encounter &y.root so it goes into a + // loop. Fail fast before this occurs. + panic("A Sparse has been copied without (*Sparse).Copy()") + } + + return root.next +} + +// IsEmpty reports whether the set s is empty. +func (s *Sparse) IsEmpty() bool { + return s.start() == &s.root +} + +// Len returns the number of elements in the set s. +func (s *Sparse) Len() int { + var l int + for b := s.start(); b != &s.root; b = b.next { + l += b.len() + } + return l +} + +// Max returns the maximum element of the set s, or MinInt if s is empty. +func (s *Sparse) Max() int { + if s.IsEmpty() { + return MinInt + } + return s.root.prev.max() +} + +// Min returns the minimum element of the set s, or MaxInt if s is empty. +func (s *Sparse) Min() int { + if s.IsEmpty() { + return MaxInt + } + return s.root.next.min(false) +} + +// block returns the block that would contain offset, +// or nil if s contains no such block. +// +func (s *Sparse) block(offset int) *block { + b := s.start() + for b != &s.root && b.offset <= offset { + if b.offset == offset { + return b + } + b = b.next + } + return nil +} + +// Insert adds x to the set s, and reports whether the set grew. +func (s *Sparse) Insert(x int) bool { + offset, i := offsetAndBitIndex(x) + b := s.start() + for b != &s.root && b.offset <= offset { + if b.offset == offset { + return b.insert(i) + } + b = b.next + } + + // Insert new block before b. + new := &block{offset: offset} + new.next = b + new.prev = b.prev + new.prev.next = new + new.next.prev = new + return new.insert(i) +} + +func (s *Sparse) removeBlock(b *block) { + b.prev.next = b.next + b.next.prev = b.prev +} + +// Remove removes x from the set s, and reports whether the set shrank. +func (s *Sparse) Remove(x int) bool { + offset, i := offsetAndBitIndex(x) + if b := s.block(offset); b != nil { + if !b.remove(i) { + return false + } + if b.empty() { + s.removeBlock(b) + } + return true + } + return false +} + +// Clear removes all elements from the set s. +func (s *Sparse) Clear() { + s.root.next = &s.root + s.root.prev = &s.root +} + +// If set s is non-empty, TakeMin sets *p to the minimum element of +// the set s, removes that element from the set and returns true. +// Otherwise, it returns false and *p is undefined. +// +// This method may be used for iteration over a worklist like so: +// +// var x int +// for worklist.TakeMin(&x) { use(x) } +// +func (s *Sparse) TakeMin(p *int) bool { + head := s.start() + if head == &s.root { + return false + } + *p = head.min(true) + if head.empty() { + s.removeBlock(head) + } + return true +} + +// Has reports whether x is an element of the set s. +func (s *Sparse) Has(x int) bool { + offset, i := offsetAndBitIndex(x) + if b := s.block(offset); b != nil { + return b.has(i) + } + return false +} + +// forEach applies function f to each element of the set s in order. +// +// f must not mutate s. Consequently, forEach is not safe to expose +// to clients. In any case, using "range s.AppendTo()" allows more +// natural control flow with continue/break/return. +// +func (s *Sparse) forEach(f func(int)) { + for b := s.start(); b != &s.root; b = b.next { + b.forEach(f) + } +} + +// Copy sets s to the value of x. +func (s *Sparse) Copy(x *Sparse) { + if s == x { + return + } + + xb := x.start() + sb := s.start() + for xb != &x.root { + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = xb.offset + sb.bits = xb.bits + xb = xb.next + sb = sb.next + } + s.discardTail(sb) +} + +// insertBlockBefore returns a new block, inserting it before next. +func (s *Sparse) insertBlockBefore(next *block) *block { + b := new(block) + b.next = next + b.prev = next.prev + b.prev.next = b + next.prev = b + return b +} + +// discardTail removes block b and all its successors from s. +func (s *Sparse) discardTail(b *block) { + if b != &s.root { + b.prev.next = &s.root + s.root.prev = b.prev + } +} + +// IntersectionWith sets s to the intersection s ∩ x. +func (s *Sparse) IntersectionWith(x *Sparse) { + if s == x { + return + } + + xb := x.start() + sb := s.start() + for xb != &x.root && sb != &s.root { + switch { + case xb.offset < sb.offset: + xb = xb.next + + case xb.offset > sb.offset: + sb = sb.next + s.removeBlock(sb.prev) + + default: + var sum word + for i := range sb.bits { + r := xb.bits[i] & sb.bits[i] + sb.bits[i] = r + sum |= r + } + if sum != 0 { + sb = sb.next + } else { + // sb will be overwritten or removed + } + + xb = xb.next + } + } + + s.discardTail(sb) +} + +// Intersection sets s to the intersection x ∩ y. +func (s *Sparse) Intersection(x, y *Sparse) { + switch { + case s == x: + s.IntersectionWith(y) + return + case s == y: + s.IntersectionWith(x) + return + case x == y: + s.Copy(x) + return + } + + xb := x.start() + yb := y.start() + sb := s.start() + for xb != &x.root && yb != &y.root { + switch { + case xb.offset < yb.offset: + xb = xb.next + continue + case xb.offset > yb.offset: + yb = yb.next + continue + } + + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = xb.offset + + var sum word + for i := range sb.bits { + r := xb.bits[i] & yb.bits[i] + sb.bits[i] = r + sum |= r + } + if sum != 0 { + sb = sb.next + } else { + // sb will be overwritten or removed + } + + xb = xb.next + yb = yb.next + } + + s.discardTail(sb) +} + +// Intersects reports whether s ∩ x ≠ ∅. +func (s *Sparse) Intersects(x *Sparse) bool { + sb := s.start() + xb := x.start() + for sb != &s.root && xb != &x.root { + switch { + case xb.offset < sb.offset: + xb = xb.next + case xb.offset > sb.offset: + sb = sb.next + default: + for i := range sb.bits { + if sb.bits[i]&xb.bits[i] != 0 { + return true + } + } + sb = sb.next + xb = xb.next + } + } + return false +} + +// UnionWith sets s to the union s ∪ x, and reports whether s grew. +func (s *Sparse) UnionWith(x *Sparse) bool { + if s == x { + return false + } + + var changed bool + xb := x.start() + sb := s.start() + for xb != &x.root { + if sb != &s.root && sb.offset == xb.offset { + for i := range xb.bits { + if sb.bits[i] != xb.bits[i] { + sb.bits[i] |= xb.bits[i] + changed = true + } + } + xb = xb.next + } else if sb == &s.root || sb.offset > xb.offset { + sb = s.insertBlockBefore(sb) + sb.offset = xb.offset + sb.bits = xb.bits + changed = true + + xb = xb.next + } + sb = sb.next + } + return changed +} + +// Union sets s to the union x ∪ y. +func (s *Sparse) Union(x, y *Sparse) { + switch { + case x == y: + s.Copy(x) + return + case s == x: + s.UnionWith(y) + return + case s == y: + s.UnionWith(x) + return + } + + xb := x.start() + yb := y.start() + sb := s.start() + for xb != &x.root || yb != &y.root { + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + switch { + case yb == &y.root || (xb != &x.root && xb.offset < yb.offset): + sb.offset = xb.offset + sb.bits = xb.bits + xb = xb.next + + case xb == &x.root || (yb != &y.root && yb.offset < xb.offset): + sb.offset = yb.offset + sb.bits = yb.bits + yb = yb.next + + default: + sb.offset = xb.offset + for i := range xb.bits { + sb.bits[i] = xb.bits[i] | yb.bits[i] + } + xb = xb.next + yb = yb.next + } + sb = sb.next + } + + s.discardTail(sb) +} + +// DifferenceWith sets s to the difference s ∖ x. +func (s *Sparse) DifferenceWith(x *Sparse) { + if s == x { + s.Clear() + return + } + + xb := x.start() + sb := s.start() + for xb != &x.root && sb != &s.root { + switch { + case xb.offset > sb.offset: + sb = sb.next + + case xb.offset < sb.offset: + xb = xb.next + + default: + var sum word + for i := range sb.bits { + r := sb.bits[i] & ^xb.bits[i] + sb.bits[i] = r + sum |= r + } + sb = sb.next + xb = xb.next + + if sum == 0 { + s.removeBlock(sb.prev) + } + } + } +} + +// Difference sets s to the difference x ∖ y. +func (s *Sparse) Difference(x, y *Sparse) { + switch { + case x == y: + s.Clear() + return + case s == x: + s.DifferenceWith(y) + return + case s == y: + var y2 Sparse + y2.Copy(y) + s.Difference(x, &y2) + return + } + + xb := x.start() + yb := y.start() + sb := s.start() + for xb != &x.root && yb != &y.root { + if xb.offset > yb.offset { + // y has block, x has none + yb = yb.next + continue + } + + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = xb.offset + + switch { + case xb.offset < yb.offset: + // x has block, y has none + sb.bits = xb.bits + + sb = sb.next + + default: + // x and y have corresponding blocks + var sum word + for i := range sb.bits { + r := xb.bits[i] & ^yb.bits[i] + sb.bits[i] = r + sum |= r + } + if sum != 0 { + sb = sb.next + } else { + // sb will be overwritten or removed + } + + yb = yb.next + } + xb = xb.next + } + + for xb != &x.root { + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = xb.offset + sb.bits = xb.bits + sb = sb.next + + xb = xb.next + } + + s.discardTail(sb) +} + +// SymmetricDifferenceWith sets s to the symmetric difference s ∆ x. +func (s *Sparse) SymmetricDifferenceWith(x *Sparse) { + if s == x { + s.Clear() + return + } + + sb := s.start() + xb := x.start() + for xb != &x.root && sb != &s.root { + switch { + case sb.offset < xb.offset: + sb = sb.next + case xb.offset < sb.offset: + nb := s.insertBlockBefore(sb) + nb.offset = xb.offset + nb.bits = xb.bits + xb = xb.next + default: + var sum word + for i := range sb.bits { + r := sb.bits[i] ^ xb.bits[i] + sb.bits[i] = r + sum |= r + } + sb = sb.next + xb = xb.next + if sum == 0 { + s.removeBlock(sb.prev) + } + } + } + + for xb != &x.root { // append the tail of x to s + sb = s.insertBlockBefore(sb) + sb.offset = xb.offset + sb.bits = xb.bits + sb = sb.next + xb = xb.next + } +} + +// SymmetricDifference sets s to the symmetric difference x ∆ y. +func (s *Sparse) SymmetricDifference(x, y *Sparse) { + switch { + case x == y: + s.Clear() + return + case s == x: + s.SymmetricDifferenceWith(y) + return + case s == y: + s.SymmetricDifferenceWith(x) + return + } + + sb := s.start() + xb := x.start() + yb := y.start() + for xb != &x.root && yb != &y.root { + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + switch { + case yb.offset < xb.offset: + sb.offset = yb.offset + sb.bits = yb.bits + sb = sb.next + yb = yb.next + case xb.offset < yb.offset: + sb.offset = xb.offset + sb.bits = xb.bits + sb = sb.next + xb = xb.next + default: + var sum word + for i := range sb.bits { + r := xb.bits[i] ^ yb.bits[i] + sb.bits[i] = r + sum |= r + } + if sum != 0 { + sb.offset = xb.offset + sb = sb.next + } + xb = xb.next + yb = yb.next + } + } + + for xb != &x.root { // append the tail of x to s + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = xb.offset + sb.bits = xb.bits + sb = sb.next + xb = xb.next + } + + for yb != &y.root { // append the tail of y to s + if sb == &s.root { + sb = s.insertBlockBefore(sb) + } + sb.offset = yb.offset + sb.bits = yb.bits + sb = sb.next + yb = yb.next + } + + s.discardTail(sb) +} + +// SubsetOf reports whether s ∖ x = ∅. +func (s *Sparse) SubsetOf(x *Sparse) bool { + if s == x { + return true + } + + sb := s.start() + xb := x.start() + for sb != &s.root { + switch { + case xb == &x.root || xb.offset > sb.offset: + return false + case xb.offset < sb.offset: + xb = xb.next + default: + for i := range sb.bits { + if sb.bits[i]&^xb.bits[i] != 0 { + return false + } + } + sb = sb.next + xb = xb.next + } + } + return true +} + +// Equals reports whether the sets s and t have the same elements. +func (s *Sparse) Equals(t *Sparse) bool { + if s == t { + return true + } + sb := s.start() + tb := t.start() + for { + switch { + case sb == &s.root && tb == &t.root: + return true + case sb == &s.root || tb == &t.root: + return false + case sb.offset != tb.offset: + return false + case sb.bits != tb.bits: + return false + } + + sb = sb.next + tb = tb.next + } +} + +// String returns a human-readable description of the set s. +func (s *Sparse) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + s.forEach(func(x int) { + if buf.Len() > 1 { + buf.WriteByte(' ') + } + fmt.Fprintf(&buf, "%d", x) + }) + buf.WriteByte('}') + return buf.String() +} + +// BitString returns the set as a string of 1s and 0s denoting the sum +// of the i'th powers of 2, for each i in s. A radix point, always +// preceded by a digit, appears if the sum is non-integral. +// +// Examples: +// {}.BitString() = "0" +// {4,5}.BitString() = "110000" +// {-3}.BitString() = "0.001" +// {-3,0,4,5}.BitString() = "110001.001" +// +func (s *Sparse) BitString() string { + if s.IsEmpty() { + return "0" + } + + min, max := s.Min(), s.Max() + var nbytes int + if max > 0 { + nbytes = max + } + nbytes++ // zero bit + radix := nbytes + if min < 0 { + nbytes += len(".") - min + } + + b := make([]byte, nbytes) + for i := range b { + b[i] = '0' + } + if radix < nbytes { + b[radix] = '.' + } + s.forEach(func(x int) { + if x >= 0 { + x += len(".") + } + b[radix-x] = '1' + }) + return string(b) +} + +// GoString returns a string showing the internal representation of +// the set s. +// +func (s *Sparse) GoString() string { + var buf bytes.Buffer + for b := s.start(); b != &s.root; b = b.next { + fmt.Fprintf(&buf, "block %p {offset=%d next=%p prev=%p", + b, b.offset, b.next, b.prev) + for _, w := range b.bits { + fmt.Fprintf(&buf, " 0%016x", w) + } + fmt.Fprintf(&buf, "}\n") + } + return buf.String() +} + +// AppendTo returns the result of appending the elements of s to slice +// in order. +func (s *Sparse) AppendTo(slice []int) []int { + s.forEach(func(x int) { + slice = append(slice, x) + }) + return slice +} + +// -- Testing/debugging ------------------------------------------------ + +// check returns an error if the representation invariants of s are violated. +func (s *Sparse) check() error { + if !s.root.empty() { + return fmt.Errorf("non-empty root block") + } + if s.root.offset != 0 { + return fmt.Errorf("root block has non-zero offset %d", s.root.offset) + } + for b := s.start(); b != &s.root; b = b.next { + if b.offset%bitsPerBlock != 0 { + return fmt.Errorf("bad offset modulo: %d", b.offset) + } + if b.empty() { + return fmt.Errorf("empty block") + } + if b.prev.next != b { + return fmt.Errorf("bad prev.next link") + } + if b.next.prev != b { + return fmt.Errorf("bad next.prev link") + } + if b.prev != &s.root { + if b.offset <= b.prev.offset { + return fmt.Errorf("bad offset order: b.offset=%d, prev.offset=%d", + b.offset, b.prev.offset) + } + } + } + return nil +} diff --git a/vendor/golang.org/x/tools/container/intsets/util.go b/vendor/golang.org/x/tools/container/intsets/util.go new file mode 100644 index 00000000000..dd1db86b1c1 --- /dev/null +++ b/vendor/golang.org/x/tools/container/intsets/util.go @@ -0,0 +1,84 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package intsets + +// From Hacker's Delight, fig 5.2. +func popcountHD(x uint32) int { + x -= (x >> 1) & 0x55555555 + x = (x & 0x33333333) + ((x >> 2) & 0x33333333) + x = (x + (x >> 4)) & 0x0f0f0f0f + x = x + (x >> 8) + x = x + (x >> 16) + return int(x & 0x0000003f) +} + +var a [1 << 8]byte + +func init() { + for i := range a { + var n byte + for x := i; x != 0; x >>= 1 { + if x&1 != 0 { + n++ + } + } + a[i] = n + } +} + +func popcountTable(x word) int { + return int(a[byte(x>>(0*8))] + + a[byte(x>>(1*8))] + + a[byte(x>>(2*8))] + + a[byte(x>>(3*8))] + + a[byte(x>>(4*8))] + + a[byte(x>>(5*8))] + + a[byte(x>>(6*8))] + + a[byte(x>>(7*8))]) +} + +// nlz returns the number of leading zeros of x. +// From Hacker's Delight, fig 5.11. +func nlz(x word) int { + x |= (x >> 1) + x |= (x >> 2) + x |= (x >> 4) + x |= (x >> 8) + x |= (x >> 16) + x |= (x >> 32) + return popcount(^x) +} + +// ntz returns the number of trailing zeros of x. +// From Hacker's Delight, fig 5.13. +func ntz(x word) int { + if x == 0 { + return bitsPerWord + } + n := 1 + if bitsPerWord == 64 { + if (x & 0xffffffff) == 0 { + n = n + 32 + x = x >> 32 + } + } + if (x & 0x0000ffff) == 0 { + n = n + 16 + x = x >> 16 + } + if (x & 0x000000ff) == 0 { + n = n + 8 + x = x >> 8 + } + if (x & 0x0000000f) == 0 { + n = n + 4 + x = x >> 4 + } + if (x & 0x00000003) == 0 { + n = n + 2 + x = x >> 2 + } + return n - int(x&1) +} From b1ac3140465306f9a66b04700082400ce81adb3b Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 19 May 2017 00:06:33 -0400 Subject: [PATCH 2/9] bump(github.com/gonum/graph): 50b27dea7ebbfb052dfaf91681afc6fde28d8796 --- third_party/BUILD | 1 + third_party/forked/gonum/graph/BUILD | 32 +++ third_party/forked/gonum/graph/LICENSE | 23 ++ third_party/forked/gonum/graph/README.md | 1 + third_party/forked/gonum/graph/graph.go | 153 +++++++++++ .../forked/gonum/graph/internal/linear/BUILD | 28 ++ .../gonum/graph/internal/linear/linear.go | 74 ++++++ third_party/forked/gonum/graph/simple/BUILD | 43 ++++ .../forked/gonum/graph/simple/simple.go | 45 ++++ .../forked/gonum/graph/simple/undirected.go | 241 ++++++++++++++++++ .../gonum/graph/simple/undirected_test.go | 63 +++++ third_party/forked/gonum/graph/traverse/BUILD | 32 +++ .../forked/gonum/graph/traverse/traverse.go | 186 ++++++++++++++ 13 files changed, 922 insertions(+) create mode 100644 third_party/forked/gonum/graph/BUILD create mode 100644 third_party/forked/gonum/graph/LICENSE create mode 100644 third_party/forked/gonum/graph/README.md create mode 100644 third_party/forked/gonum/graph/graph.go create mode 100644 third_party/forked/gonum/graph/internal/linear/BUILD create mode 100644 third_party/forked/gonum/graph/internal/linear/linear.go create mode 100644 third_party/forked/gonum/graph/simple/BUILD create mode 100644 third_party/forked/gonum/graph/simple/simple.go create mode 100644 third_party/forked/gonum/graph/simple/undirected.go create mode 100644 third_party/forked/gonum/graph/simple/undirected_test.go create mode 100644 third_party/forked/gonum/graph/traverse/BUILD create mode 100644 third_party/forked/gonum/graph/traverse/traverse.go diff --git a/third_party/BUILD b/third_party/BUILD index 801e43d5501..a100a2ca1ae 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -20,6 +20,7 @@ filegroup( "//third_party/forked/golang/expansion:all-srcs", "//third_party/forked/golang/reflect:all-srcs", "//third_party/forked/golang/template:all-srcs", + "//third_party/forked/gonum/graph:all-srcs", "//third_party/htpasswd:all-srcs", ], tags = ["automanaged"], diff --git a/third_party/forked/gonum/graph/BUILD b/third_party/forked/gonum/graph/BUILD new file mode 100644 index 00000000000..5fef90777fb --- /dev/null +++ b/third_party/forked/gonum/graph/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["graph.go"], + tags = ["automanaged"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//third_party/forked/gonum/graph/internal/linear:all-srcs", + "//third_party/forked/gonum/graph/simple:all-srcs", + "//third_party/forked/gonum/graph/traverse:all-srcs", + ], + tags = ["automanaged"], +) diff --git a/third_party/forked/gonum/graph/LICENSE b/third_party/forked/gonum/graph/LICENSE new file mode 100644 index 00000000000..76edf5ef7e9 --- /dev/null +++ b/third_party/forked/gonum/graph/LICENSE @@ -0,0 +1,23 @@ +Copyright ©2013 The gonum Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the gonum project nor the names of its authors and + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/third_party/forked/gonum/graph/README.md b/third_party/forked/gonum/graph/README.md new file mode 100644 index 00000000000..3b51e664a47 --- /dev/null +++ b/third_party/forked/gonum/graph/README.md @@ -0,0 +1 @@ +Forked from gonum/graph@50b27dea7ebbfb052dfaf91681afc6fde28d8796 to support memory-use improvements to the simple graph diff --git a/third_party/forked/gonum/graph/graph.go b/third_party/forked/gonum/graph/graph.go new file mode 100644 index 00000000000..adade5d79bb --- /dev/null +++ b/third_party/forked/gonum/graph/graph.go @@ -0,0 +1,153 @@ +// Copyright ©2014 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package graph + +// Node is a graph node. It returns a graph-unique integer ID. +type Node interface { + ID() int +} + +// Edge is a graph edge. In directed graphs, the direction of the +// edge is given from -> to, otherwise the edge is semantically +// unordered. +type Edge interface { + From() Node + To() Node + Weight() float64 +} + +// Graph is a generalized graph. +type Graph interface { + // Has returns whether the node exists within the graph. + Has(Node) bool + + // Nodes returns all the nodes in the graph. + Nodes() []Node + + // From returns all nodes that can be reached directly + // from the given node. + From(Node) []Node + + // HasEdgeBeteen returns whether an edge exists between + // nodes x and y without considering direction. + HasEdgeBetween(x, y Node) bool + + // Edge returns the edge from u to v if such an edge + // exists and nil otherwise. The node v must be directly + // reachable from u as defined by the From method. + Edge(u, v Node) Edge +} + +// Undirected is an undirected graph. +type Undirected interface { + Graph + + // EdgeBetween returns the edge between nodes x and y. + EdgeBetween(x, y Node) Edge +} + +// Directed is a directed graph. +type Directed interface { + Graph + + // HasEdgeFromTo returns whether an edge exists + // in the graph from u to v. + HasEdgeFromTo(u, v Node) bool + + // To returns all nodes that can reach directly + // to the given node. + To(Node) []Node +} + +// Weighter defines graphs that can report edge weights. +type Weighter interface { + // Weight returns the weight for the edge between + // x and y if Edge(x, y) returns a non-nil Edge. + // If x and y are the same node or there is no + // joining edge between the two nodes the weight + // value returned is implementation dependent. + // Weight returns true if an edge exists between + // x and y or if x and y have the same ID, false + // otherwise. + Weight(x, y Node) (w float64, ok bool) +} + +// NodeAdder is an interface for adding arbitrary nodes to a graph. +type NodeAdder interface { + // NewNodeID returns a new unique arbitrary ID. + NewNodeID() int + + // Adds a node to the graph. AddNode panics if + // the added node ID matches an existing node ID. + AddNode(Node) +} + +// NodeRemover is an interface for removing nodes from a graph. +type NodeRemover interface { + // RemoveNode removes a node from the graph, as + // well as any edges attached to it. If the node + // is not in the graph it is a no-op. + RemoveNode(Node) +} + +// EdgeSetter is an interface for adding edges to a graph. +type EdgeSetter interface { + // SetEdge adds an edge from one node to another. + // If the graph supports node addition the nodes + // will be added if they do not exist, otherwise + // SetEdge will panic. + // If the IDs returned by e.From and e.To are + // equal, SetEdge will panic. + SetEdge(e Edge) +} + +// EdgeRemover is an interface for removing nodes from a graph. +type EdgeRemover interface { + // RemoveEdge removes the given edge, leaving the + // terminal nodes. If the edge does not exist it + // is a no-op. + RemoveEdge(Edge) +} + +// Builder is a graph that can have nodes and edges added. +type Builder interface { + NodeAdder + EdgeSetter +} + +// UndirectedBuilder is an undirected graph builder. +type UndirectedBuilder interface { + Undirected + Builder +} + +// DirectedBuilder is a directed graph builder. +type DirectedBuilder interface { + Directed + Builder +} + +// Copy copies nodes and edges as undirected edges from the source to the destination +// without first clearing the destination. Copy will panic if a node ID in the source +// graph matches a node ID in the destination. +// +// If the source is undirected and the destination is directed both directions will +// be present in the destination after the copy is complete. +// +// If the source is a directed graph, the destination is undirected, and a fundamental +// cycle exists with two nodes where the edge weights differ, the resulting destination +// graph's edge weight between those nodes is undefined. If there is a defined function +// to resolve such conflicts, an Undirect may be used to do this. +func Copy(dst Builder, src Graph) { + nodes := src.Nodes() + for _, n := range nodes { + dst.AddNode(n) + } + for _, u := range nodes { + for _, v := range src.From(u) { + dst.SetEdge(src.Edge(u, v)) + } + } +} diff --git a/third_party/forked/gonum/graph/internal/linear/BUILD b/third_party/forked/gonum/graph/internal/linear/BUILD new file mode 100644 index 00000000000..95b086fd354 --- /dev/null +++ b/third_party/forked/gonum/graph/internal/linear/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["linear.go"], + tags = ["automanaged"], + deps = ["//third_party/forked/gonum/graph:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/third_party/forked/gonum/graph/internal/linear/linear.go b/third_party/forked/gonum/graph/internal/linear/linear.go new file mode 100644 index 00000000000..ce7c6cfffdf --- /dev/null +++ b/third_party/forked/gonum/graph/internal/linear/linear.go @@ -0,0 +1,74 @@ +// Copyright ©2015 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package linear provides common linear data structures. +package linear + +import ( + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +// NodeStack implements a LIFO stack of graph.Node. +type NodeStack []graph.Node + +// Len returns the number of graph.Nodes on the stack. +func (s *NodeStack) Len() int { return len(*s) } + +// Pop returns the last graph.Node on the stack and removes it +// from the stack. +func (s *NodeStack) Pop() graph.Node { + v := *s + v, n := v[:len(v)-1], v[len(v)-1] + *s = v + return n +} + +// Push adds the node n to the stack at the last position. +func (s *NodeStack) Push(n graph.Node) { *s = append(*s, n) } + +// NodeQueue implements a FIFO queue. +type NodeQueue struct { + head int + data []graph.Node +} + +// Len returns the number of graph.Nodes in the queue. +func (q *NodeQueue) Len() int { return len(q.data) - q.head } + +// Enqueue adds the node n to the back of the queue. +func (q *NodeQueue) Enqueue(n graph.Node) { + if len(q.data) == cap(q.data) && q.head > 0 { + l := q.Len() + copy(q.data, q.data[q.head:]) + q.head = 0 + q.data = append(q.data[:l], n) + } else { + q.data = append(q.data, n) + } +} + +// Dequeue returns the graph.Node at the front of the queue and +// removes it from the queue. +func (q *NodeQueue) Dequeue() graph.Node { + if q.Len() == 0 { + panic("queue: empty queue") + } + + var n graph.Node + n, q.data[q.head] = q.data[q.head], nil + q.head++ + + if q.Len() == 0 { + q.head = 0 + q.data = q.data[:0] + } + + return n +} + +// Reset clears the queue for reuse. +func (q *NodeQueue) Reset() { + q.head = 0 + q.data = q.data[:0] +} diff --git a/third_party/forked/gonum/graph/simple/BUILD b/third_party/forked/gonum/graph/simple/BUILD new file mode 100644 index 00000000000..a5c9b446a4b --- /dev/null +++ b/third_party/forked/gonum/graph/simple/BUILD @@ -0,0 +1,43 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["undirected_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//third_party/forked/gonum/graph:go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = [ + "simple.go", + "undirected.go", + ], + tags = ["automanaged"], + deps = [ + "//third_party/forked/gonum/graph:go_default_library", + "//vendor/golang.org/x/tools/container/intsets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/third_party/forked/gonum/graph/simple/simple.go b/third_party/forked/gonum/graph/simple/simple.go new file mode 100644 index 00000000000..9bc56b8be63 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/simple.go @@ -0,0 +1,45 @@ +// Copyright ©2014 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package simple provides a suite of simple graph implementations satisfying +// the gonum/graph interfaces. +package simple + +import ( + "math" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +// Node is a simple graph node. +type Node int + +// ID returns the ID number of the node. +func (n Node) ID() int { + return int(n) +} + +// Edge is a simple graph edge. +type Edge struct { + F, T graph.Node + W float64 +} + +// From returns the from-node of the edge. +func (e Edge) From() graph.Node { return e.F } + +// To returns the to-node of the edge. +func (e Edge) To() graph.Node { return e.T } + +// Weight returns the weight of the edge. +func (e Edge) Weight() float64 { return e.W } + +// maxInt is the maximum value of the machine-dependent int type. +const maxInt int = int(^uint(0) >> 1) + +// isSame returns whether two float64 values are the same where NaN values +// are equalable. +func isSame(a, b float64) bool { + return a == b || (math.IsNaN(a) && math.IsNaN(b)) +} diff --git a/third_party/forked/gonum/graph/simple/undirected.go b/third_party/forked/gonum/graph/simple/undirected.go new file mode 100644 index 00000000000..1d08d0cc271 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/undirected.go @@ -0,0 +1,241 @@ +// Copyright ©2014 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package simple + +import ( + "fmt" + + "golang.org/x/tools/container/intsets" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +// UndirectedGraph implements a generalized undirected graph. +type UndirectedGraph struct { + nodes map[int]graph.Node + edges map[int]map[int]graph.Edge + + self, absent float64 + + freeIDs intsets.Sparse + usedIDs intsets.Sparse +} + +// NewUndirectedGraph returns an UndirectedGraph with the specified self and absent +// edge weight values. +func NewUndirectedGraph(self, absent float64) *UndirectedGraph { + return &UndirectedGraph{ + nodes: make(map[int]graph.Node), + edges: make(map[int]map[int]graph.Edge), + + self: self, + absent: absent, + } +} + +// NewNodeID returns a new unique ID for a node to be added to g. The returned ID does +// not become a valid ID in g until it is added to g. +func (g *UndirectedGraph) NewNodeID() int { + if len(g.nodes) == 0 { + return 0 + } + if len(g.nodes) == maxInt { + panic(fmt.Sprintf("simple: cannot allocate node: no slot")) + } + + var id int + if g.freeIDs.Len() != 0 && g.freeIDs.TakeMin(&id) { + return id + } + if id = g.usedIDs.Max(); id < maxInt { + return id + 1 + } + for id = 0; id < maxInt; id++ { + if !g.usedIDs.Has(id) { + return id + } + } + panic("unreachable") +} + +// AddNode adds n to the graph. It panics if the added node ID matches an existing node ID. +func (g *UndirectedGraph) AddNode(n graph.Node) { + if _, exists := g.nodes[n.ID()]; exists { + panic(fmt.Sprintf("simple: node ID collision: %d", n.ID())) + } + g.nodes[n.ID()] = n + g.edges[n.ID()] = make(map[int]graph.Edge) + + g.freeIDs.Remove(n.ID()) + g.usedIDs.Insert(n.ID()) +} + +// RemoveNode removes n from the graph, as well as any edges attached to it. If the node +// is not in the graph it is a no-op. +func (g *UndirectedGraph) RemoveNode(n graph.Node) { + if _, ok := g.nodes[n.ID()]; !ok { + return + } + delete(g.nodes, n.ID()) + + for from := range g.edges[n.ID()] { + delete(g.edges[from], n.ID()) + } + delete(g.edges, n.ID()) + + g.freeIDs.Insert(n.ID()) + g.usedIDs.Remove(n.ID()) + +} + +// SetEdge adds e, an edge from one node to another. If the nodes do not exist, they are added. +// It will panic if the IDs of the e.From and e.To are equal. +func (g *UndirectedGraph) SetEdge(e graph.Edge) { + var ( + from = e.From() + fid = from.ID() + to = e.To() + tid = to.ID() + ) + + if fid == tid { + panic("simple: adding self edge") + } + + if !g.Has(from) { + g.AddNode(from) + } + if !g.Has(to) { + g.AddNode(to) + } + + g.edges[fid][tid] = e + g.edges[tid][fid] = e +} + +// RemoveEdge removes e from the graph, leaving the terminal nodes. If the edge does not exist +// it is a no-op. +func (g *UndirectedGraph) RemoveEdge(e graph.Edge) { + from, to := e.From(), e.To() + if _, ok := g.nodes[from.ID()]; !ok { + return + } + if _, ok := g.nodes[to.ID()]; !ok { + return + } + + delete(g.edges[from.ID()], to.ID()) + delete(g.edges[to.ID()], from.ID()) +} + +// Node returns the node in the graph with the given ID. +func (g *UndirectedGraph) Node(id int) graph.Node { + return g.nodes[id] +} + +// Has returns whether the node exists within the graph. +func (g *UndirectedGraph) Has(n graph.Node) bool { + _, ok := g.nodes[n.ID()] + return ok +} + +// Nodes returns all the nodes in the graph. +func (g *UndirectedGraph) Nodes() []graph.Node { + nodes := make([]graph.Node, len(g.nodes)) + i := 0 + for _, n := range g.nodes { + nodes[i] = n + i++ + } + + return nodes +} + +// Edges returns all the edges in the graph. +func (g *UndirectedGraph) Edges() []graph.Edge { + var edges []graph.Edge + + seen := make(map[[2]int]struct{}) + for _, u := range g.edges { + for _, e := range u { + uid := e.From().ID() + vid := e.To().ID() + if _, ok := seen[[2]int{uid, vid}]; ok { + continue + } + seen[[2]int{uid, vid}] = struct{}{} + seen[[2]int{vid, uid}] = struct{}{} + edges = append(edges, e) + } + } + + return edges +} + +// From returns all nodes in g that can be reached directly from n. +func (g *UndirectedGraph) From(n graph.Node) []graph.Node { + if !g.Has(n) { + return nil + } + + nodes := make([]graph.Node, len(g.edges[n.ID()])) + i := 0 + for from := range g.edges[n.ID()] { + nodes[i] = g.nodes[from] + i++ + } + + return nodes +} + +// HasEdgeBetween returns whether an edge exists between nodes x and y. +func (g *UndirectedGraph) HasEdgeBetween(x, y graph.Node) bool { + _, ok := g.edges[x.ID()][y.ID()] + return ok +} + +// Edge returns the edge from u to v if such an edge exists and nil otherwise. +// The node v must be directly reachable from u as defined by the From method. +func (g *UndirectedGraph) Edge(u, v graph.Node) graph.Edge { + return g.EdgeBetween(u, v) +} + +// EdgeBetween returns the edge between nodes x and y. +func (g *UndirectedGraph) EdgeBetween(x, y graph.Node) graph.Edge { + // We don't need to check if neigh exists because + // it's implicit in the edges access. + if !g.Has(x) { + return nil + } + + return g.edges[x.ID()][y.ID()] +} + +// Weight returns the weight for the edge between x and y if Edge(x, y) returns a non-nil Edge. +// If x and y are the same node or there is no joining edge between the two nodes the weight +// value returned is either the graph's absent or self value. Weight returns true if an edge +// exists between x and y or if x and y have the same ID, false otherwise. +func (g *UndirectedGraph) Weight(x, y graph.Node) (w float64, ok bool) { + xid := x.ID() + yid := y.ID() + if xid == yid { + return g.self, true + } + if n, ok := g.edges[xid]; ok { + if e, ok := n[yid]; ok { + return e.Weight(), true + } + } + return g.absent, false +} + +// Degree returns the degree of n in g. +func (g *UndirectedGraph) Degree(n graph.Node) int { + if _, ok := g.nodes[n.ID()]; !ok { + return 0 + } + + return len(g.edges[n.ID()]) +} diff --git a/third_party/forked/gonum/graph/simple/undirected_test.go b/third_party/forked/gonum/graph/simple/undirected_test.go new file mode 100644 index 00000000000..42c1f606170 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/undirected_test.go @@ -0,0 +1,63 @@ +// Copyright ©2014 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package simple + +import ( + "math" + "testing" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +var _ graph.Graph = (*UndirectedGraph)(nil) + +func TestAssertMutableNotDirected(t *testing.T) { + var g graph.UndirectedBuilder = NewUndirectedGraph(0, math.Inf(1)) + if _, ok := g.(graph.Directed); ok { + t.Fatal("Graph is directed, but a MutableGraph cannot safely be directed!") + } +} + +func TestMaxID(t *testing.T) { + g := NewUndirectedGraph(0, math.Inf(1)) + nodes := make(map[graph.Node]struct{}) + for i := Node(0); i < 3; i++ { + g.AddNode(i) + nodes[i] = struct{}{} + } + g.RemoveNode(Node(0)) + delete(nodes, Node(0)) + g.RemoveNode(Node(2)) + delete(nodes, Node(2)) + n := Node(g.NewNodeID()) + g.AddNode(n) + if !g.Has(n) { + t.Error("added node does not exist in graph") + } + if _, exists := nodes[n]; exists { + t.Errorf("Created already existing node id: %v", n.ID()) + } +} + +// Test for issue #123 https://github.com/gonum/graph/issues/123 +func TestIssue123UndirectedGraph(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("unexpected panic: %v", r) + } + }() + g := NewUndirectedGraph(0, math.Inf(1)) + + n0 := Node(g.NewNodeID()) + g.AddNode(n0) + + n1 := Node(g.NewNodeID()) + g.AddNode(n1) + + g.RemoveNode(n0) + + n2 := Node(g.NewNodeID()) + g.AddNode(n2) +} diff --git a/third_party/forked/gonum/graph/traverse/BUILD b/third_party/forked/gonum/graph/traverse/BUILD new file mode 100644 index 00000000000..ec40a55dd58 --- /dev/null +++ b/third_party/forked/gonum/graph/traverse/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["traverse.go"], + tags = ["automanaged"], + deps = [ + "//third_party/forked/gonum/graph:go_default_library", + "//third_party/forked/gonum/graph/internal/linear:go_default_library", + "//vendor/golang.org/x/tools/container/intsets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/third_party/forked/gonum/graph/traverse/traverse.go b/third_party/forked/gonum/graph/traverse/traverse.go new file mode 100644 index 00000000000..cc361c85fe3 --- /dev/null +++ b/third_party/forked/gonum/graph/traverse/traverse.go @@ -0,0 +1,186 @@ +// Copyright ©2015 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traverse provides basic graph traversal primitives. +package traverse + +import ( + "golang.org/x/tools/container/intsets" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" + "k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear" +) + +// BreadthFirst implements stateful breadth-first graph traversal. +type BreadthFirst struct { + EdgeFilter func(graph.Edge) bool + Visit func(u, v graph.Node) + queue linear.NodeQueue + visited *intsets.Sparse +} + +// Walk performs a breadth-first traversal of the graph g starting from the given node, +// depending on the the EdgeFilter field and the until parameter if they are non-nil. The +// traversal follows edges for which EdgeFilter(edge) is true and returns the first node +// for which until(node, depth) is true. During the traversal, if the Visit field is +// non-nil, it is called with the nodes joined by each followed edge. +func (b *BreadthFirst) Walk(g graph.Graph, from graph.Node, until func(n graph.Node, d int) bool) graph.Node { + if b.visited == nil { + b.visited = &intsets.Sparse{} + } + b.queue.Enqueue(from) + b.visited.Insert(from.ID()) + + var ( + depth int + children int + untilNext = 1 + ) + for b.queue.Len() > 0 { + t := b.queue.Dequeue() + if until != nil && until(t, depth) { + return t + } + for _, n := range g.From(t) { + if b.EdgeFilter != nil && !b.EdgeFilter(g.Edge(t, n)) { + continue + } + if b.visited.Has(n.ID()) { + continue + } + if b.Visit != nil { + b.Visit(t, n) + } + b.visited.Insert(n.ID()) + children++ + b.queue.Enqueue(n) + } + if untilNext--; untilNext == 0 { + depth++ + untilNext = children + children = 0 + } + } + + return nil +} + +// WalkAll calls Walk for each unvisited node of the graph g using edges independent +// of their direction. The functions before and after are called prior to commencing +// and after completing each walk if they are non-nil respectively. The function +// during is called on each node as it is traversed. +func (b *BreadthFirst) WalkAll(g graph.Undirected, before, after func(), during func(graph.Node)) { + b.Reset() + for _, from := range g.Nodes() { + if b.Visited(from) { + continue + } + if before != nil { + before() + } + b.Walk(g, from, func(n graph.Node, _ int) bool { + if during != nil { + during(n) + } + return false + }) + if after != nil { + after() + } + } +} + +// Visited returned whether the node n was visited during a traverse. +func (b *BreadthFirst) Visited(n graph.Node) bool { + return b.visited != nil && b.visited.Has(n.ID()) +} + +// Reset resets the state of the traverser for reuse. +func (b *BreadthFirst) Reset() { + b.queue.Reset() + if b.visited != nil { + b.visited.Clear() + } +} + +// DepthFirst implements stateful depth-first graph traversal. +type DepthFirst struct { + EdgeFilter func(graph.Edge) bool + Visit func(u, v graph.Node) + stack linear.NodeStack + visited *intsets.Sparse +} + +// Walk performs a depth-first traversal of the graph g starting from the given node, +// depending on the the EdgeFilter field and the until parameter if they are non-nil. The +// traversal follows edges for which EdgeFilter(edge) is true and returns the first node +// for which until(node) is true. During the traversal, if the Visit field is non-nil, it +// is called with the nodes joined by each followed edge. +func (d *DepthFirst) Walk(g graph.Graph, from graph.Node, until func(graph.Node) bool) graph.Node { + if d.visited == nil { + d.visited = &intsets.Sparse{} + } + d.stack.Push(from) + d.visited.Insert(from.ID()) + + for d.stack.Len() > 0 { + t := d.stack.Pop() + if until != nil && until(t) { + return t + } + for _, n := range g.From(t) { + if d.EdgeFilter != nil && !d.EdgeFilter(g.Edge(t, n)) { + continue + } + if d.visited.Has(n.ID()) { + continue + } + if d.Visit != nil { + d.Visit(t, n) + } + d.visited.Insert(n.ID()) + d.stack.Push(n) + } + } + + return nil +} + +// WalkAll calls Walk for each unvisited node of the graph g using edges independent +// of their direction. The functions before and after are called prior to commencing +// and after completing each walk if they are non-nil respectively. The function +// during is called on each node as it is traversed. +func (d *DepthFirst) WalkAll(g graph.Undirected, before, after func(), during func(graph.Node)) { + d.Reset() + for _, from := range g.Nodes() { + if d.Visited(from) { + continue + } + if before != nil { + before() + } + d.Walk(g, from, func(n graph.Node) bool { + if during != nil { + during(n) + } + return false + }) + if after != nil { + after() + } + } +} + +// Visited returned whether the node n was visited during a traverse. +func (d *DepthFirst) Visited(n graph.Node) bool { + return d.visited != nil && d.visited.Has(n.ID()) +} + +// Reset resets the state of the traverser for reuse. +func (d *DepthFirst) Reset() { + d.stack = d.stack[:0] + if d.visited != nil { + d.visited.Clear() + } +} From ec1d79da191c33e5b16b58e808880d522d3313d2 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 22 May 2017 21:15:14 -0400 Subject: [PATCH 3/9] gonum: directed acyclic graph Implements graph.Directed capable of storing at most one edge between any two nodes. Uses the Undirected implementation for space efficiency (~30% space savings). --- third_party/forked/gonum/graph/simple/BUILD | 6 +- .../gonum/graph/simple/directed_acyclic.go | 55 ++++++++++++++++ .../graph/simple/directed_acyclic_test.go | 62 +++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 third_party/forked/gonum/graph/simple/directed_acyclic.go create mode 100644 third_party/forked/gonum/graph/simple/directed_acyclic_test.go diff --git a/third_party/forked/gonum/graph/simple/BUILD b/third_party/forked/gonum/graph/simple/BUILD index a5c9b446a4b..b4f8db1897b 100644 --- a/third_party/forked/gonum/graph/simple/BUILD +++ b/third_party/forked/gonum/graph/simple/BUILD @@ -10,7 +10,10 @@ load( go_test( name = "go_default_test", - srcs = ["undirected_test.go"], + srcs = [ + "directed_acyclic_test.go", + "undirected_test.go", + ], library = ":go_default_library", tags = ["automanaged"], deps = ["//third_party/forked/gonum/graph:go_default_library"], @@ -19,6 +22,7 @@ go_test( go_library( name = "go_default_library", srcs = [ + "directed_acyclic.go", "simple.go", "undirected.go", ], diff --git a/third_party/forked/gonum/graph/simple/directed_acyclic.go b/third_party/forked/gonum/graph/simple/directed_acyclic.go new file mode 100644 index 00000000000..20ed2f658a9 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/directed_acyclic.go @@ -0,0 +1,55 @@ +package simple + +import ( + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +// DirectedAcyclicGraph implements graph.Directed using UndirectedGraph, +// which only stores one edge for any node pair. +type DirectedAcyclicGraph struct { + *UndirectedGraph +} + +func NewDirectedAcyclicGraph(self, absent float64) *DirectedAcyclicGraph { + return &DirectedAcyclicGraph{ + UndirectedGraph: NewUndirectedGraph(self, absent), + } +} + +func (g *DirectedAcyclicGraph) HasEdgeFromTo(u, v graph.Node) bool { + edge := g.UndirectedGraph.EdgeBetween(u, v) + if edge == nil { + return false + } + return (edge.From().ID() == u.ID()) +} + +func (g *DirectedAcyclicGraph) From(n graph.Node) []graph.Node { + if !g.Has(n) { + return nil + } + + fid := n.ID() + nodes := make([]graph.Node, 0, len(g.UndirectedGraph.edges[n.ID()])) + for _, edge := range g.UndirectedGraph.edges[n.ID()] { + if edge.From().ID() == fid { + nodes = append(nodes, g.UndirectedGraph.nodes[edge.To().ID()]) + } + } + return nodes +} + +func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node { + if !g.Has(n) { + return nil + } + + tid := n.ID() + nodes := make([]graph.Node, 0, len(g.UndirectedGraph.edges[n.ID()])) + for _, edge := range g.UndirectedGraph.edges[n.ID()] { + if edge.To().ID() == tid { + nodes = append(nodes, g.UndirectedGraph.nodes[edge.From().ID()]) + } + } + return nodes +} diff --git a/third_party/forked/gonum/graph/simple/directed_acyclic_test.go b/third_party/forked/gonum/graph/simple/directed_acyclic_test.go new file mode 100644 index 00000000000..0f3454d8a71 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/directed_acyclic_test.go @@ -0,0 +1,62 @@ +// Copyright ©2014 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package simple + +import ( + "math" + "testing" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +var _ graph.Graph = &DirectedAcyclicGraph{} +var _ graph.Directed = &DirectedAcyclicGraph{} + +// Tests Issue #27 +func TestAcyclicEdgeOvercounting(t *testing.T) { + g := generateDummyAcyclicGraph() + + if neigh := g.From(Node(Node(2))); len(neigh) != 2 { + t.Errorf("Node 2 has incorrect number of neighbors got neighbors %v (count %d), expected 2 neighbors {0,1}", neigh, len(neigh)) + } +} + +func generateDummyAcyclicGraph() *DirectedAcyclicGraph { + nodes := [4]struct{ srcID, targetID int }{ + {2, 1}, + {1, 0}, + {0, 2}, + {2, 0}, + } + + g := NewDirectedAcyclicGraph(0, math.Inf(1)) + + for _, n := range nodes { + g.SetEdge(Edge{F: Node(n.srcID), T: Node(n.targetID), W: 1}) + } + + return g +} + +// Test for issue #123 https://github.com/gonum/graph/issues/123 +func TestAcyclicIssue123DirectedGraph(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("unexpected panic: %v", r) + } + }() + g := NewDirectedAcyclicGraph(0, math.Inf(1)) + + n0 := Node(g.NewNodeID()) + g.AddNode(n0) + + n1 := Node(g.NewNodeID()) + g.AddNode(n1) + + g.RemoveNode(n0) + + n2 := Node(g.NewNodeID()) + g.AddNode(n2) +} From 385b84ad83e67faee7f88a0938acfa85b7da044e Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 22 May 2017 21:16:43 -0400 Subject: [PATCH 4/9] gonum: visiting graph traversal Use visitors to avoid allocating slices to hold nodes during traversal (and to allow short-circuiting) Benchmarked at 95% space savings traversing nodes with many edges. --- .../gonum/graph/simple/directed_acyclic.go | 28 ++++++ third_party/forked/gonum/graph/traverse/BUILD | 5 +- .../gonum/graph/traverse/visit_depth_first.go | 86 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 third_party/forked/gonum/graph/traverse/visit_depth_first.go diff --git a/third_party/forked/gonum/graph/simple/directed_acyclic.go b/third_party/forked/gonum/graph/simple/directed_acyclic.go index 20ed2f658a9..bd4da5b16cb 100644 --- a/third_party/forked/gonum/graph/simple/directed_acyclic.go +++ b/third_party/forked/gonum/graph/simple/directed_acyclic.go @@ -39,6 +39,20 @@ func (g *DirectedAcyclicGraph) From(n graph.Node) []graph.Node { return nodes } +func (g *DirectedAcyclicGraph) VisitFrom(n graph.Node, visitor func(neighbor graph.Node) (shouldContinue bool)) { + if !g.Has(n) { + return + } + fid := n.ID() + for _, edge := range g.UndirectedGraph.edges[n.ID()] { + if edge.From().ID() == fid { + if !visitor(g.UndirectedGraph.nodes[edge.To().ID()]) { + return + } + } + } +} + func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node { if !g.Has(n) { return nil @@ -53,3 +67,17 @@ func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node { } return nodes } + +func (g *DirectedAcyclicGraph) VisitTo(n graph.Node, visitor func(neighbor graph.Node) (shouldContinue bool)) { + if !g.Has(n) { + return + } + tid := n.ID() + for _, edge := range g.UndirectedGraph.edges[n.ID()] { + if edge.To().ID() == tid { + if !visitor(g.UndirectedGraph.nodes[edge.From().ID()]) { + return + } + } + } +} diff --git a/third_party/forked/gonum/graph/traverse/BUILD b/third_party/forked/gonum/graph/traverse/BUILD index ec40a55dd58..e0eb285a761 100644 --- a/third_party/forked/gonum/graph/traverse/BUILD +++ b/third_party/forked/gonum/graph/traverse/BUILD @@ -9,7 +9,10 @@ load( go_library( name = "go_default_library", - srcs = ["traverse.go"], + srcs = [ + "traverse.go", + "visit_depth_first.go", + ], tags = ["automanaged"], deps = [ "//third_party/forked/gonum/graph:go_default_library", diff --git a/third_party/forked/gonum/graph/traverse/visit_depth_first.go b/third_party/forked/gonum/graph/traverse/visit_depth_first.go new file mode 100644 index 00000000000..b7f45a7b321 --- /dev/null +++ b/third_party/forked/gonum/graph/traverse/visit_depth_first.go @@ -0,0 +1,86 @@ +// Copyright ©2015 The gonum Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traverse provides basic graph traversal primitives. +package traverse + +import ( + "golang.org/x/tools/container/intsets" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" + "k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear" +) + +// VisitableGraph +type VisitableGraph interface { + graph.Graph + + // VisitFrom invokes visitor with all nodes that can be reached directly from the given node. + // If visitor returns false, visiting is short-circuited. + VisitFrom(from graph.Node, visitor func(graph.Node) (shouldContinue bool)) +} + +// VisitingDepthFirst implements stateful depth-first graph traversal on a visitable graph. +type VisitingDepthFirst struct { + EdgeFilter func(graph.Edge) bool + Visit func(u, v graph.Node) + stack linear.NodeStack + visited *intsets.Sparse +} + +// Walk performs a depth-first traversal of the graph g starting from the given node, +// depending on the the EdgeFilter field and the until parameter if they are non-nil. The +// traversal follows edges for which EdgeFilter(edge) is true and returns the first node +// for which until(node) is true. During the traversal, if the Visit field is non-nil, it +// is called with the nodes joined by each followed edge. +func (d *VisitingDepthFirst) Walk(g VisitableGraph, from graph.Node, until func(graph.Node) bool) graph.Node { + if d.visited == nil { + d.visited = &intsets.Sparse{} + } + d.stack.Push(from) + d.visited.Insert(from.ID()) + if until != nil && until(from) { + return from + } + + var found graph.Node + for d.stack.Len() > 0 { + t := d.stack.Pop() + g.VisitFrom(t, func(n graph.Node) (shouldContinue bool) { + if d.EdgeFilter != nil && !d.EdgeFilter(g.Edge(t, n)) { + return true + } + if d.visited.Has(n.ID()) { + return true + } + if d.Visit != nil { + d.Visit(t, n) + } + d.visited.Insert(n.ID()) + d.stack.Push(n) + if until != nil && until(n) { + found = n + return false + } + return true + }) + if found != nil { + return found + } + } + return nil +} + +// Visited returned whether the node n was visited during a traverse. +func (d *VisitingDepthFirst) Visited(n graph.Node) bool { + return d.visited != nil && d.visited.Has(n.ID()) +} + +// Reset resets the state of the traverser for reuse. +func (d *VisitingDepthFirst) Reset() { + d.stack = d.stack[:0] + if d.visited != nil { + d.visited.Clear() + } +} From 2d2427b847f528b4ec60c665af56237c09bc33f8 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 22 May 2017 15:51:47 -0400 Subject: [PATCH 5/9] gonum: slice-based edge holder Use slices to store sets of edges for n <= 4. ~20% memory savings with many nodes with n=2 edges. --- third_party/forked/gonum/graph/simple/BUILD | 2 + .../gonum/graph/simple/directed_acyclic.go | 20 +-- .../forked/gonum/graph/simple/edgeholder.go | 122 ++++++++++++++++++ .../gonum/graph/simple/edgeholder_test.go | 104 +++++++++++++++ .../forked/gonum/graph/simple/undirected.go | 43 +++--- 5 files changed, 260 insertions(+), 31 deletions(-) create mode 100644 third_party/forked/gonum/graph/simple/edgeholder.go create mode 100644 third_party/forked/gonum/graph/simple/edgeholder_test.go diff --git a/third_party/forked/gonum/graph/simple/BUILD b/third_party/forked/gonum/graph/simple/BUILD index b4f8db1897b..092f84cb823 100644 --- a/third_party/forked/gonum/graph/simple/BUILD +++ b/third_party/forked/gonum/graph/simple/BUILD @@ -12,6 +12,7 @@ go_test( name = "go_default_test", srcs = [ "directed_acyclic_test.go", + "edgeholder_test.go", "undirected_test.go", ], library = ":go_default_library", @@ -23,6 +24,7 @@ go_library( name = "go_default_library", srcs = [ "directed_acyclic.go", + "edgeholder.go", "simple.go", "undirected.go", ], diff --git a/third_party/forked/gonum/graph/simple/directed_acyclic.go b/third_party/forked/gonum/graph/simple/directed_acyclic.go index bd4da5b16cb..ac930feb169 100644 --- a/third_party/forked/gonum/graph/simple/directed_acyclic.go +++ b/third_party/forked/gonum/graph/simple/directed_acyclic.go @@ -30,12 +30,12 @@ func (g *DirectedAcyclicGraph) From(n graph.Node) []graph.Node { } fid := n.ID() - nodes := make([]graph.Node, 0, len(g.UndirectedGraph.edges[n.ID()])) - for _, edge := range g.UndirectedGraph.edges[n.ID()] { + nodes := make([]graph.Node, 0, g.UndirectedGraph.edges[n.ID()].Len()) + g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { if edge.From().ID() == fid { nodes = append(nodes, g.UndirectedGraph.nodes[edge.To().ID()]) } - } + }) return nodes } @@ -44,13 +44,13 @@ func (g *DirectedAcyclicGraph) VisitFrom(n graph.Node, visitor func(neighbor gra return } fid := n.ID() - for _, edge := range g.UndirectedGraph.edges[n.ID()] { + g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { if edge.From().ID() == fid { if !visitor(g.UndirectedGraph.nodes[edge.To().ID()]) { return } } - } + }) } func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node { @@ -59,12 +59,12 @@ func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node { } tid := n.ID() - nodes := make([]graph.Node, 0, len(g.UndirectedGraph.edges[n.ID()])) - for _, edge := range g.UndirectedGraph.edges[n.ID()] { + nodes := make([]graph.Node, 0, g.UndirectedGraph.edges[n.ID()].Len()) + g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { if edge.To().ID() == tid { nodes = append(nodes, g.UndirectedGraph.nodes[edge.From().ID()]) } - } + }) return nodes } @@ -73,11 +73,11 @@ func (g *DirectedAcyclicGraph) VisitTo(n graph.Node, visitor func(neighbor graph return } tid := n.ID() - for _, edge := range g.UndirectedGraph.edges[n.ID()] { + g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { if edge.To().ID() == tid { if !visitor(g.UndirectedGraph.nodes[edge.From().ID()]) { return } } - } + }) } diff --git a/third_party/forked/gonum/graph/simple/edgeholder.go b/third_party/forked/gonum/graph/simple/edgeholder.go new file mode 100644 index 00000000000..f2248ab7db9 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/edgeholder.go @@ -0,0 +1,122 @@ +package simple + +import "k8s.io/kubernetes/third_party/forked/gonum/graph" + +// edgeHolder represents a set of edges, with no more than one edge to or from a particular neighbor node +type edgeHolder interface { + // Visit invokes visitor with each edge and the id of the neighbor node in the edge + Visit(visitor func(neighbor int, edge graph.Edge)) + // Delete removes edges to or from the specified neighbor + Delete(neighbor int) edgeHolder + // Set stores the edge to or from the specified neighbor + Set(neighbor int, edge graph.Edge) edgeHolder + // Get returns the edge to or from the specified neighbor + Get(neighbor int) (graph.Edge, bool) + // Len returns the number of edges + Len() int +} + +// sliceEdgeHolder holds a list of edges to or from self +type sliceEdgeHolder struct { + self int + edges []graph.Edge +} + +func (e *sliceEdgeHolder) Visit(visitor func(neighbor int, edge graph.Edge)) { + for _, edge := range e.edges { + if edge.From().ID() == e.self { + visitor(edge.To().ID(), edge) + } else { + visitor(edge.From().ID(), edge) + } + } +} +func (e *sliceEdgeHolder) Delete(neighbor int) edgeHolder { + edges := e.edges[:0] + for i, edge := range e.edges { + if edge.From().ID() == e.self { + if edge.To().ID() == neighbor { + continue + } + } else { + if edge.From().ID() == neighbor { + continue + } + } + edges = append(edges, e.edges[i]) + } + e.edges = edges + return e +} +func (e *sliceEdgeHolder) Set(neighbor int, newEdge graph.Edge) edgeHolder { + for i, edge := range e.edges { + if edge.From().ID() == e.self { + if edge.To().ID() == neighbor { + e.edges[i] = newEdge + return e + } + } else { + if edge.From().ID() == neighbor { + e.edges[i] = newEdge + return e + } + } + } + + if len(e.edges) < 4 { + e.edges = append(e.edges, newEdge) + return e + } + + h := mapEdgeHolder(make(map[int]graph.Edge, len(e.edges)+1)) + for i, edge := range e.edges { + if edge.From().ID() == e.self { + h[edge.To().ID()] = e.edges[i] + } else { + h[edge.From().ID()] = e.edges[i] + } + } + h[neighbor] = newEdge + return h +} +func (e *sliceEdgeHolder) Get(neighbor int) (graph.Edge, bool) { + for _, edge := range e.edges { + if edge.From().ID() == e.self { + if edge.To().ID() == neighbor { + return edge, true + } + } else { + if edge.From().ID() == neighbor { + return edge, true + } + } + } + return nil, false +} +func (e *sliceEdgeHolder) Len() int { + return len(e.edges) +} + +// mapEdgeHolder holds a map of neighbors to edges +type mapEdgeHolder map[int]graph.Edge + +func (e mapEdgeHolder) Visit(visitor func(neighbor int, edge graph.Edge)) { + for neighbor, edge := range e { + visitor(neighbor, edge) + } +} +func (e mapEdgeHolder) Delete(neighbor int) edgeHolder { + delete(e, neighbor) + return e +} +func (e mapEdgeHolder) Set(neighbor int, edge graph.Edge) edgeHolder { + e[neighbor] = edge + return e +} +func (e mapEdgeHolder) Get(neighbor int) (graph.Edge, bool) { + edge, ok := e[neighbor] + return edge, ok +} +func (e mapEdgeHolder) Len() int { + return len(e) +} diff --git a/third_party/forked/gonum/graph/simple/edgeholder_test.go b/third_party/forked/gonum/graph/simple/edgeholder_test.go new file mode 100644 index 00000000000..6c9be375000 --- /dev/null +++ b/third_party/forked/gonum/graph/simple/edgeholder_test.go @@ -0,0 +1,104 @@ +package simple + +import ( + "reflect" + "sort" + "testing" + + "k8s.io/kubernetes/third_party/forked/gonum/graph" +) + +func TestEdgeHolder(t *testing.T) { + holder := edgeHolder(&sliceEdgeHolder{self: 1}) + + // Empty tests + if len := holder.Len(); len != 0 { + t.Errorf("expected 0") + } + if n, ok := holder.Get(2); ok || n != nil { + t.Errorf("expected nil,false") + } + holder.Visit(func(_ int, _ graph.Edge) { t.Errorf("unexpected call to visitor") }) + holder = holder.Delete(2) + + // Insert an edge to ourselves + holder = holder.Set(1, Edge{F: Node(1), T: Node(1)}) + if len := holder.Len(); len != 1 { + t.Errorf("expected 1") + } + if n, ok := holder.Get(1); !ok || n == nil || n.From().ID() != 1 || n.To().ID() != 1 { + t.Errorf("expected edge to ourselves, got %#v", n) + } + neighbors := []int{} + holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) }) + if !reflect.DeepEqual(neighbors, []int{1}) { + t.Errorf("expected a single visit to ourselves, got %v", neighbors) + } + + // Insert edges from us to other nodes + holder = holder.Set(2, Edge{F: Node(1), T: Node(2)}) + holder = holder.Set(3, Edge{F: Node(1), T: Node(3)}) + holder = holder.Set(4, Edge{F: Node(1), T: Node(4)}) + if len := holder.Len(); len != 4 { + t.Errorf("expected 4") + } + if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 1 || n.To().ID() != 2 { + t.Errorf("expected edge from us to another node, got %#v", n) + } + neighbors = []int{} + holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) }) + if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4}) { + t.Errorf("expected a single visit to ourselves, got %v", neighbors) + } + + // Insert edges to us to other nodes + holder = holder.Set(2, Edge{F: Node(2), T: Node(1)}) + holder = holder.Set(3, Edge{F: Node(3), T: Node(1)}) + holder = holder.Set(4, Edge{F: Node(4), T: Node(1)}) + if len := holder.Len(); len != 4 { + t.Errorf("expected 4") + } + if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 2 || n.To().ID() != 1 { + t.Errorf("expected reversed edge, got %#v", n) + } + neighbors = []int{} + holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) }) + if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4}) { + t.Errorf("expected a single visit to ourselves, got %v", neighbors) + } + + if _, ok := holder.(*sliceEdgeHolder); !ok { + t.Errorf("expected slice edge holder") + } + + // Make the transition to a map + holder = holder.Set(5, Edge{F: Node(5), T: Node(1)}) + + if _, ok := holder.(mapEdgeHolder); !ok { + t.Errorf("expected map edge holder") + } + if len := holder.Len(); len != 5 { + t.Errorf("expected 5") + } + if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 2 || n.To().ID() != 1 { + t.Errorf("expected old edges, got %#v", n) + } + if n, ok := holder.Get(5); !ok || n == nil || n.From().ID() != 5 || n.To().ID() != 1 { + t.Errorf("expected new edge, got %#v", n) + } + neighbors = []int{} + holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) }) + sort.Ints(neighbors) // sort, map order is random + if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4, 5}) { + t.Errorf("expected 1,2,3,4,5, got %v", neighbors) + } + holder = holder.Delete(1) + holder = holder.Delete(2) + holder = holder.Delete(3) + holder = holder.Delete(4) + holder = holder.Delete(5) + holder = holder.Delete(6) + if len := holder.Len(); len != 0 { + t.Errorf("expected 0") + } +} diff --git a/third_party/forked/gonum/graph/simple/undirected.go b/third_party/forked/gonum/graph/simple/undirected.go index 1d08d0cc271..231fa3deda4 100644 --- a/third_party/forked/gonum/graph/simple/undirected.go +++ b/third_party/forked/gonum/graph/simple/undirected.go @@ -15,7 +15,7 @@ import ( // UndirectedGraph implements a generalized undirected graph. type UndirectedGraph struct { nodes map[int]graph.Node - edges map[int]map[int]graph.Edge + edges map[int]edgeHolder self, absent float64 @@ -28,7 +28,7 @@ type UndirectedGraph struct { func NewUndirectedGraph(self, absent float64) *UndirectedGraph { return &UndirectedGraph{ nodes: make(map[int]graph.Node), - edges: make(map[int]map[int]graph.Edge), + edges: make(map[int]edgeHolder), self: self, absent: absent, @@ -66,7 +66,7 @@ func (g *UndirectedGraph) AddNode(n graph.Node) { panic(fmt.Sprintf("simple: node ID collision: %d", n.ID())) } g.nodes[n.ID()] = n - g.edges[n.ID()] = make(map[int]graph.Edge) + g.edges[n.ID()] = &sliceEdgeHolder{self: n.ID()} g.freeIDs.Remove(n.ID()) g.usedIDs.Insert(n.ID()) @@ -80,9 +80,9 @@ func (g *UndirectedGraph) RemoveNode(n graph.Node) { } delete(g.nodes, n.ID()) - for from := range g.edges[n.ID()] { - delete(g.edges[from], n.ID()) - } + g.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { + g.edges[neighbor] = g.edges[neighbor].Delete(n.ID()) + }) delete(g.edges, n.ID()) g.freeIDs.Insert(n.ID()) @@ -111,8 +111,8 @@ func (g *UndirectedGraph) SetEdge(e graph.Edge) { g.AddNode(to) } - g.edges[fid][tid] = e - g.edges[tid][fid] = e + g.edges[fid] = g.edges[fid].Set(tid, e) + g.edges[tid] = g.edges[tid].Set(fid, e) } // RemoveEdge removes e from the graph, leaving the terminal nodes. If the edge does not exist @@ -126,8 +126,8 @@ func (g *UndirectedGraph) RemoveEdge(e graph.Edge) { return } - delete(g.edges[from.ID()], to.ID()) - delete(g.edges[to.ID()], from.ID()) + g.edges[from.ID()] = g.edges[from.ID()].Delete(to.ID()) + g.edges[to.ID()] = g.edges[to.ID()].Delete(from.ID()) } // Node returns the node in the graph with the given ID. @@ -159,16 +159,16 @@ func (g *UndirectedGraph) Edges() []graph.Edge { seen := make(map[[2]int]struct{}) for _, u := range g.edges { - for _, e := range u { + u.Visit(func(neighbor int, e graph.Edge) { uid := e.From().ID() vid := e.To().ID() if _, ok := seen[[2]int{uid, vid}]; ok { - continue + return } seen[[2]int{uid, vid}] = struct{}{} seen[[2]int{vid, uid}] = struct{}{} edges = append(edges, e) - } + }) } return edges @@ -180,19 +180,19 @@ func (g *UndirectedGraph) From(n graph.Node) []graph.Node { return nil } - nodes := make([]graph.Node, len(g.edges[n.ID()])) + nodes := make([]graph.Node, g.edges[n.ID()].Len()) i := 0 - for from := range g.edges[n.ID()] { - nodes[i] = g.nodes[from] + g.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) { + nodes[i] = g.nodes[neighbor] i++ - } + }) return nodes } // HasEdgeBetween returns whether an edge exists between nodes x and y. func (g *UndirectedGraph) HasEdgeBetween(x, y graph.Node) bool { - _, ok := g.edges[x.ID()][y.ID()] + _, ok := g.edges[x.ID()].Get(y.ID()) return ok } @@ -210,7 +210,8 @@ func (g *UndirectedGraph) EdgeBetween(x, y graph.Node) graph.Edge { return nil } - return g.edges[x.ID()][y.ID()] + edge, _ := g.edges[x.ID()].Get(y.ID()) + return edge } // Weight returns the weight for the edge between x and y if Edge(x, y) returns a non-nil Edge. @@ -224,7 +225,7 @@ func (g *UndirectedGraph) Weight(x, y graph.Node) (w float64, ok bool) { return g.self, true } if n, ok := g.edges[xid]; ok { - if e, ok := n[yid]; ok { + if e, ok := n.Get(yid); ok { return e.Weight(), true } } @@ -237,5 +238,5 @@ func (g *UndirectedGraph) Degree(n graph.Node) int { return 0 } - return len(g.edges[n.ID()]) + return g.edges[n.ID()].Len() } From 48a9357926f906af581602b1f79ad43fef42e0a4 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 5 May 2017 02:14:27 -0400 Subject: [PATCH 6/9] Add PV util for extracting referenced secrets --- hack/.linted_packages | 1 + pkg/api/BUILD | 1 + pkg/api/persistentvolume/BUILD | 41 ++++++++ pkg/api/persistentvolume/OWNERS | 4 + pkg/api/persistentvolume/util.go | 55 +++++++++++ pkg/api/persistentvolume/util_test.go | 133 ++++++++++++++++++++++++++ 6 files changed, 235 insertions(+) create mode 100644 pkg/api/persistentvolume/BUILD create mode 100755 pkg/api/persistentvolume/OWNERS create mode 100644 pkg/api/persistentvolume/util.go create mode 100644 pkg/api/persistentvolume/util_test.go diff --git a/hack/.linted_packages b/hack/.linted_packages index f0f818422d9..6f787605487 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -59,6 +59,7 @@ pkg/api/errors pkg/api/events pkg/api/install pkg/api/meta +pkg/api/persistentvolume pkg/api/pod pkg/api/resource pkg/api/service diff --git a/pkg/api/BUILD b/pkg/api/BUILD index c4ca2b63398..ee993082031 100644 --- a/pkg/api/BUILD +++ b/pkg/api/BUILD @@ -107,6 +107,7 @@ filegroup( "//pkg/api/helper:all-srcs", "//pkg/api/install:all-srcs", "//pkg/api/meta:all-srcs", + "//pkg/api/persistentvolume:all-srcs", "//pkg/api/pod:all-srcs", "//pkg/api/ref:all-srcs", "//pkg/api/resource:all-srcs", diff --git a/pkg/api/persistentvolume/BUILD b/pkg/api/persistentvolume/BUILD new file mode 100644 index 00000000000..1699f870c30 --- /dev/null +++ b/pkg/api/persistentvolume/BUILD @@ -0,0 +1,41 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = ["util.go"], + tags = ["automanaged"], + deps = ["//pkg/api:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +go_test( + name = "go_default_test", + srcs = ["util_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + ], +) diff --git a/pkg/api/persistentvolume/OWNERS b/pkg/api/persistentvolume/OWNERS new file mode 100755 index 00000000000..b74bd2bf49a --- /dev/null +++ b/pkg/api/persistentvolume/OWNERS @@ -0,0 +1,4 @@ +reviewers: +- smarterclayton +- kargakis +- david-mcmahon diff --git a/pkg/api/persistentvolume/util.go b/pkg/api/persistentvolume/util.go new file mode 100644 index 00000000000..b33077d68da --- /dev/null +++ b/pkg/api/persistentvolume/util.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package persistentvolume + +import ( + "k8s.io/kubernetes/pkg/api" +) + +// VisitPVSecretNames invokes the visitor function with the name of every secret +// referenced by the PV spec. If visitor returns false, visiting is short-circuited. +// Returns true if visiting completed, false if visiting was short-circuited. +func VisitPVSecretNames(pv *api.PersistentVolume, visitor func(string) bool) bool { + source := &pv.Spec.PersistentVolumeSource + switch { + case source.AzureFile != nil: + if len(source.AzureFile.SecretName) > 0 && !visitor(source.AzureFile.SecretName) { + return false + } + case source.CephFS != nil: + if source.CephFS.SecretRef != nil && !visitor(source.CephFS.SecretRef.Name) { + return false + } + case source.FlexVolume != nil: + if source.FlexVolume.SecretRef != nil && !visitor(source.FlexVolume.SecretRef.Name) { + return false + } + case source.RBD != nil: + if source.RBD.SecretRef != nil && !visitor(source.RBD.SecretRef.Name) { + return false + } + case source.ScaleIO != nil: + if source.ScaleIO.SecretRef != nil && !visitor(source.ScaleIO.SecretRef.Name) { + return false + } + case source.ISCSI != nil: + if source.ISCSI.SecretRef != nil && !visitor(source.ISCSI.SecretRef.Name) { + return false + } + } + return true +} diff --git a/pkg/api/persistentvolume/util_test.go b/pkg/api/persistentvolume/util_test.go new file mode 100644 index 00000000000..f8353050de5 --- /dev/null +++ b/pkg/api/persistentvolume/util_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package persistentvolume + +import ( + "reflect" + "testing" + + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/kubernetes/pkg/api" +) + +func TestPVSecrets(t *testing.T) { + // Stub containing all possible secret references in a PV. + // The names of the referenced secrets match struct paths detected by reflection. + pvs := []*api.PersistentVolume{ + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + AzureFile: &api.AzureFileVolumeSource{ + SecretName: "Spec.PersistentVolumeSource.AzureFile.SecretName"}}}}, + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + CephFS: &api.CephFSVolumeSource{ + SecretRef: &api.LocalObjectReference{ + Name: "Spec.PersistentVolumeSource.CephFS.SecretRef"}}}}}, + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + FlexVolume: &api.FlexVolumeSource{ + SecretRef: &api.LocalObjectReference{ + Name: "Spec.PersistentVolumeSource.FlexVolume.SecretRef"}}}}}, + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + RBD: &api.RBDVolumeSource{ + SecretRef: &api.LocalObjectReference{ + Name: "Spec.PersistentVolumeSource.RBD.SecretRef"}}}}}, + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + ScaleIO: &api.ScaleIOVolumeSource{ + SecretRef: &api.LocalObjectReference{ + Name: "Spec.PersistentVolumeSource.ScaleIO.SecretRef"}}}}}, + {Spec: api.PersistentVolumeSpec{PersistentVolumeSource: api.PersistentVolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + SecretRef: &api.LocalObjectReference{ + Name: "Spec.PersistentVolumeSource.ISCSI.SecretRef"}}}}}, + } + extractedNames := sets.NewString() + for _, pv := range pvs { + VisitPVSecretNames(pv, func(name string) bool { + extractedNames.Insert(name) + return true + }) + } + + // excludedSecretPaths holds struct paths to fields with "secret" in the name that are not actually references to secret API objects + excludedSecretPaths := sets.NewString( + "Spec.PersistentVolumeSource.CephFS.SecretFile", + ) + // expectedSecretPaths holds struct paths to fields with "secret" in the name that are references to secret API objects. + // every path here should be represented as an example in the PV stub above, with the secret name set to the path. + expectedSecretPaths := sets.NewString( + "Spec.PersistentVolumeSource.AzureFile.SecretName", + "Spec.PersistentVolumeSource.CephFS.SecretRef", + "Spec.PersistentVolumeSource.FlexVolume.SecretRef", + "Spec.PersistentVolumeSource.RBD.SecretRef", + "Spec.PersistentVolumeSource.ScaleIO.SecretRef", + "Spec.PersistentVolumeSource.ISCSI.SecretRef", + ) + secretPaths := collectSecretPaths(t, nil, "", reflect.TypeOf(&api.PersistentVolume{})) + secretPaths = secretPaths.Difference(excludedSecretPaths) + if missingPaths := expectedSecretPaths.Difference(secretPaths); len(missingPaths) > 0 { + t.Logf("Missing expected secret paths:\n%s", strings.Join(missingPaths.List(), "\n")) + t.Error("Missing expected secret paths. Verify VisitPVSecretNames() is correctly finding the missing paths, then correct expectedSecretPaths") + } + if extraPaths := secretPaths.Difference(expectedSecretPaths); len(extraPaths) > 0 { + t.Logf("Extra secret paths:\n%s", strings.Join(extraPaths.List(), "\n")) + t.Error("Extra fields with 'secret' in the name found. Verify VisitPVSecretNames() is including these fields if appropriate, then correct expectedSecretPaths") + } + + if missingNames := expectedSecretPaths.Difference(extractedNames); len(missingNames) > 0 { + t.Logf("Missing expected secret names:\n%s", strings.Join(missingNames.List(), "\n")) + t.Error("Missing expected secret names. Verify the PV stub above includes these references, then verify VisitPVSecretNames() is correctly finding the missing names") + } + if extraNames := extractedNames.Difference(expectedSecretPaths); len(extraNames) > 0 { + t.Logf("Extra secret names:\n%s", strings.Join(extraNames.List(), "\n")) + t.Error("Extra secret names extracted. Verify VisitPVSecretNames() is correctly extracting secret names") + } +} + +// collectSecretPaths traverses the object, computing all the struct paths that lead to fields with "secret" in the name. +func collectSecretPaths(t *testing.T, path *field.Path, name string, tp reflect.Type) sets.String { + secretPaths := sets.NewString() + + if tp.Kind() == reflect.Ptr { + secretPaths.Insert(collectSecretPaths(t, path, name, tp.Elem()).List()...) + return secretPaths + } + + if strings.Contains(strings.ToLower(name), "secret") { + secretPaths.Insert(path.String()) + } + + switch tp.Kind() { + case reflect.Ptr: + secretPaths.Insert(collectSecretPaths(t, path, name, tp.Elem()).List()...) + case reflect.Struct: + for i := 0; i < tp.NumField(); i++ { + field := tp.Field(i) + secretPaths.Insert(collectSecretPaths(t, path.Child(field.Name), field.Name, field.Type).List()...) + } + case reflect.Interface: + t.Errorf("cannot find secret fields in interface{} field %s", path.String()) + case reflect.Map: + secretPaths.Insert(collectSecretPaths(t, path.Key("*"), "", tp.Elem()).List()...) + case reflect.Slice: + secretPaths.Insert(collectSecretPaths(t, path.Key("*"), "", tp.Elem()).List()...) + default: + // all primitive types + } + + return secretPaths +} From d278a80a670a7e1a86ff1729fa67c89f6cc5bf81 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 5 May 2017 02:14:02 -0400 Subject: [PATCH 7/9] Separate system:node rules --- .../authorizer/rbac/bootstrappolicy/policy.go | 86 ++++++++++--------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index caefaf38589..4e4e3a8e09c 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -86,6 +86,50 @@ func addClusterRoleBindingLabel(rolebindings []rbac.ClusterRoleBinding) { return } +func NodeRules() []rbac.PolicyRule { + return []rbac.PolicyRule{ + // Needed to check API access. These creates are non-mutating + rbac.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), + rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews", "localsubjectaccessreviews").RuleOrDie(), + // Needed to build serviceLister, to populate env vars for services + rbac.NewRule(Read...).Groups(legacyGroup).Resources("services").RuleOrDie(), + // Nodes can register themselves + // TODO: restrict to creating a node with the same name they announce + rbac.NewRule("create", "get", "list", "watch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), + // TODO: restrict to the bound node once supported + rbac.NewRule("update", "patch").Groups(legacyGroup).Resources("nodes/status").RuleOrDie(), + rbac.NewRule("update", "patch", "delete").Groups(legacyGroup).Resources("nodes").RuleOrDie(), + + // TODO: restrict to the bound node as creator once supported + rbac.NewRule("create", "update", "patch").Groups(legacyGroup).Resources("events").RuleOrDie(), + + // TODO: restrict to pods scheduled on the bound node once supported + rbac.NewRule(Read...).Groups(legacyGroup).Resources("pods").RuleOrDie(), + + // TODO: remove once mirror pods are removed + // TODO: restrict deletion to mirror pods created by the bound node once supported + // Needed for the node to create/delete mirror pods + rbac.NewRule("create", "delete").Groups(legacyGroup).Resources("pods").RuleOrDie(), + // TODO: restrict to pods scheduled on the bound node once supported + rbac.NewRule("update").Groups(legacyGroup).Resources("pods/status").RuleOrDie(), + + // TODO: restrict to secrets and configmaps used by pods scheduled on bound node once supported + // Needed for imagepullsecrets, rbd/ceph and secret volumes, and secrets in envs + // Needed for configmap volume and envs + rbac.NewRule("get").Groups(legacyGroup).Resources("secrets", "configmaps").RuleOrDie(), + // TODO: restrict to claims/volumes used by pods scheduled on bound node once supported + // Needed for persistent volumes + rbac.NewRule("get").Groups(legacyGroup).Resources("persistentvolumeclaims", "persistentvolumes").RuleOrDie(), + // TODO: restrict to namespaces of pods scheduled on bound node once supported + // TODO: change glusterfs to use DNS lookup so this isn't needed? + // Needed for glusterfs volumes + rbac.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), + // Used to create a certificatesigningrequest for a node-specific client certificate, and watch + // for it to be signed. This allows the kubelet to rotate it's own certificate. + rbac.NewRule("create", "get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), + } +} + // ClusterRoles returns the cluster roles to bootstrap an API server with func ClusterRoles() []rbac.ClusterRole { roles := []rbac.ClusterRole{ @@ -204,47 +248,7 @@ func ClusterRoles() []rbac.ClusterRole { { // a role for nodes to use to have the access they need for running pods ObjectMeta: metav1.ObjectMeta{Name: "system:node"}, - Rules: []rbac.PolicyRule{ - // Needed to check API access. These creates are non-mutating - rbac.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), - rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews", "localsubjectaccessreviews").RuleOrDie(), - // Needed to build serviceLister, to populate env vars for services - rbac.NewRule(Read...).Groups(legacyGroup).Resources("services").RuleOrDie(), - // Nodes can register themselves - // TODO: restrict to creating a node with the same name they announce - rbac.NewRule("create", "get", "list", "watch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), - // TODO: restrict to the bound node once supported - rbac.NewRule("update", "patch").Groups(legacyGroup).Resources("nodes/status").RuleOrDie(), - rbac.NewRule("update", "patch", "delete").Groups(legacyGroup).Resources("nodes").RuleOrDie(), - - // TODO: restrict to the bound node as creator once supported - rbac.NewRule("create", "update", "patch").Groups(legacyGroup).Resources("events").RuleOrDie(), - - // TODO: restrict to pods scheduled on the bound node once supported - rbac.NewRule(Read...).Groups(legacyGroup).Resources("pods").RuleOrDie(), - - // TODO: remove once mirror pods are removed - // TODO: restrict deletion to mirror pods created by the bound node once supported - // Needed for the node to create/delete mirror pods - rbac.NewRule("create", "delete").Groups(legacyGroup).Resources("pods").RuleOrDie(), - // TODO: restrict to pods scheduled on the bound node once supported - rbac.NewRule("update").Groups(legacyGroup).Resources("pods/status").RuleOrDie(), - - // TODO: restrict to secrets and configmaps used by pods scheduled on bound node once supported - // Needed for imagepullsecrets, rbd/ceph and secret volumes, and secrets in envs - // Needed for configmap volume and envs - rbac.NewRule("get").Groups(legacyGroup).Resources("secrets", "configmaps").RuleOrDie(), - // TODO: restrict to claims/volumes used by pods scheduled on bound node once supported - // Needed for persistent volumes - rbac.NewRule("get").Groups(legacyGroup).Resources("persistentvolumeclaims", "persistentvolumes").RuleOrDie(), - // TODO: restrict to namespaces of pods scheduled on bound node once supported - // TODO: change glusterfs to use DNS lookup so this isn't needed? - // Needed for glusterfs volumes - rbac.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), - // Used to create a certificatesigningrequest for a node-specific client certificate, and watch - // for it to be signed. This allows the kubelet to rotate it's own certificate. - rbac.NewRule("create", "get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), - }, + Rules: NodeRules(), }, { // a role to use for node-problem-detector access. It does not get bound to default location since From fed30040c9ec8c95d5f7438fae3b82cf517731ff Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Sat, 27 May 2017 02:33:53 -0400 Subject: [PATCH 8/9] Make NodeRestriction admission require identifiable nodes --- .../pkg/admission/noderestriction/admission.go | 16 +++++----------- .../admission/noderestriction/admission_test.go | 5 ++--- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/plugin/pkg/admission/noderestriction/admission.go b/plugin/pkg/admission/noderestriction/admission.go index 13191b59b10..9615a871c49 100644 --- a/plugin/pkg/admission/noderestriction/admission.go +++ b/plugin/pkg/admission/noderestriction/admission.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package node +package noderestriction import ( "fmt" @@ -37,24 +37,22 @@ const ( // Register registers a plugin func Register(plugins *admission.Plugins) { plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { - return NewPlugin(nodeidentifier.NewDefaultNodeIdentifier(), false), nil + return NewPlugin(nodeidentifier.NewDefaultNodeIdentifier()), nil }) } // NewPlugin creates a new NodeRestriction admission plugin. // This plugin identifies requests from nodes -func NewPlugin(nodeIdentifier nodeidentifier.NodeIdentifier, strict bool) *nodePlugin { +func NewPlugin(nodeIdentifier nodeidentifier.NodeIdentifier) *nodePlugin { return &nodePlugin{ Handler: admission.NewHandler(admission.Create, admission.Update, admission.Delete), nodeIdentifier: nodeIdentifier, - strict: strict, } } // nodePlugin holds state for and implements the admission plugin. type nodePlugin struct { *admission.Handler - strict bool nodeIdentifier nodeidentifier.NodeIdentifier podsGetter coreinternalversion.PodsGetter } @@ -92,12 +90,8 @@ func (c *nodePlugin) Admit(a admission.Attributes) error { } if len(nodeName) == 0 { - if c.strict { - // In strict mode, disallow requests from nodes we cannot match to a particular node - return admission.NewForbidden(a, fmt.Errorf("could not determine node identity from user")) - } - // Our job is just to restrict identifiable nodes - return nil + // disallow requests we cannot match to a particular node + return admission.NewForbidden(a, fmt.Errorf("could not determine node from user %s", a.GetUserInfo().GetName())) } switch a.GetResource().GroupResource() { diff --git a/plugin/pkg/admission/noderestriction/admission_test.go b/plugin/pkg/admission/noderestriction/admission_test.go index 71cea49e870..9dbc8388703 100644 --- a/plugin/pkg/admission/noderestriction/admission_test.go +++ b/plugin/pkg/admission/noderestriction/admission_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package node +package noderestriction import ( "strings" @@ -82,7 +82,6 @@ func Test_nodePlugin_Admit(t *testing.T) { tests := []struct { name string - strict bool podsGetter coreinternalversion.PodsGetter attributes admission.Attributes err string @@ -473,7 +472,7 @@ func Test_nodePlugin_Admit(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := NewPlugin(nodeidentifier.NewDefaultNodeIdentifier(), tt.strict) + c := NewPlugin(nodeidentifier.NewDefaultNodeIdentifier()) c.podsGetter = tt.podsGetter err := c.Admit(tt.attributes) if (err == nil) != (len(tt.err) == 0) { From fc8e915a4bcfdd224d1ff625569a7af1a2766690 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 30 May 2017 15:15:38 -0400 Subject: [PATCH 9/9] Add Node authorization mode based on graph of node-related objects --- hack/local-up-cluster.sh | 12 +- pkg/kubeapiserver/authorizer/BUILD | 3 + pkg/kubeapiserver/authorizer/config.go | 16 + pkg/kubeapiserver/authorizer/modes/modes.go | 3 +- plugin/pkg/auth/authorizer/BUILD | 1 + plugin/pkg/auth/authorizer/node/BUILD | 63 +++ plugin/pkg/auth/authorizer/node/OWNERS | 9 + plugin/pkg/auth/authorizer/node/graph.go | 265 +++++++++++ .../auth/authorizer/node/graph_populator.go | 108 +++++ .../auth/authorizer/node/node_authorizer.go | 164 +++++++ .../authorizer/node/node_authorizer_test.go | 437 ++++++++++++++++++ .../authorizer/rbac/bootstrappolicy/policy.go | 82 +++- test/integration/auth/BUILD | 8 + test/integration/auth/node_test.go | 303 ++++++++++++ 14 files changed, 1457 insertions(+), 17 deletions(-) create mode 100644 plugin/pkg/auth/authorizer/node/BUILD create mode 100644 plugin/pkg/auth/authorizer/node/OWNERS create mode 100644 plugin/pkg/auth/authorizer/node/graph.go create mode 100644 plugin/pkg/auth/authorizer/node/graph_populator.go create mode 100644 plugin/pkg/auth/authorizer/node/node_authorizer.go create mode 100644 plugin/pkg/auth/authorizer/node/node_authorizer_test.go create mode 100644 test/integration/auth/node_test.go diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index c59c53b64ce..28344d23825 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -76,6 +76,7 @@ ENABLE_APISERVER_BASIC_AUDIT=${ENABLE_APISERVER_BASIC_AUDIT:-false} # RBAC Mode options ALLOW_ANY_TOKEN=${ALLOW_ANY_TOKEN:-false} ENABLE_RBAC=${ENABLE_RBAC:-false} +AUTHORIZATION_MODE=${AUTHORIZATION_MODE:-""} KUBECONFIG_TOKEN=${KUBECONFIG_TOKEN:-""} AUTH_ARGS=${AUTH_ARGS:-""} @@ -201,6 +202,8 @@ KUBELET_HOST=${KUBELET_HOST:-"127.0.0.1"} API_CORS_ALLOWED_ORIGINS=${API_CORS_ALLOWED_ORIGINS:-/127.0.0.1(:[0-9]+)?$,/localhost(:[0-9]+)?$} KUBELET_PORT=${KUBELET_PORT:-10250} LOG_LEVEL=${LOG_LEVEL:-3} +# Use to increase verbosity on particular files, e.g. LOG_SPEC=token_controller*=5,other_controller*=4 +LOG_SPEC=${LOG_SPEC:-""} LOG_DIR=${LOG_DIR:-"/tmp"} CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"docker"} CONTAINER_RUNTIME_ENDPOINT=${CONTAINER_RUNTIME_ENDPOINT:-""} @@ -435,6 +438,12 @@ function start_apiserver { if [[ "${ENABLE_RBAC}" = true ]]; then authorizer_arg="--authorization-mode=RBAC " fi + if [[ -n "${AUTHORIZATION_MODE}" ]]; then + if [[ "${ENABLE_RBAC}" = true ]]; then + warning "AUTHORIZATION_MODE=$AUTHORIZATION_MODE overrode ENABLE_RBAC=true" + fi + authorizer_arg="--authorization-mode=${AUTHORIZATION_MODE} " + fi priv_arg="" if [[ -n "${ALLOW_PRIVILEGED}" ]]; then priv_arg="--allow-privileged " @@ -487,6 +496,7 @@ function start_apiserver { ${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${swagger_arg} ${audit_arg} ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\ ${advertise_address} \ --v=${LOG_LEVEL} \ + --vmodule="${LOG_SPEC}" \ --cert-dir="${CERT_DIR}" \ --client-ca-file="${CERT_DIR}/client-ca.crt" \ --service-account-key-file="${SERVICE_ACCOUNT_KEY}" \ @@ -894,7 +904,7 @@ if [[ "${START_MODE}" != "nokubelet" ]]; then esac fi -if [[ -n "${PSP_ADMISSION}" && "${ENABLE_RBAC}" = true ]]; then +if [[ -n "${PSP_ADMISSION}" && ("${ENABLE_RBAC}" = true || "${AUTHORIZATION_MODE}" = *RBAC* ) ]]; then create_psp_policy fi diff --git a/pkg/kubeapiserver/authorizer/BUILD b/pkg/kubeapiserver/authorizer/BUILD index 39708710310..95d4851d014 100644 --- a/pkg/kubeapiserver/authorizer/BUILD +++ b/pkg/kubeapiserver/authorizer/BUILD @@ -26,10 +26,13 @@ go_library( deps = [ "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", + "//pkg/auth/nodeidentifier:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/client/listers/rbac/internalversion:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", + "//plugin/pkg/auth/authorizer/node:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", + "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", diff --git a/pkg/kubeapiserver/authorizer/config.go b/pkg/kubeapiserver/authorizer/config.go index 5f5a8a74107..3a65d5d1555 100644 --- a/pkg/kubeapiserver/authorizer/config.go +++ b/pkg/kubeapiserver/authorizer/config.go @@ -28,10 +28,13 @@ import ( "k8s.io/apiserver/plugin/pkg/authorizer/webhook" rbacapi "k8s.io/kubernetes/pkg/apis/rbac" "k8s.io/kubernetes/pkg/auth/authorizer/abac" + "k8s.io/kubernetes/pkg/auth/nodeidentifier" informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" rbaclisters "k8s.io/kubernetes/pkg/client/listers/rbac/internalversion" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/node" "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" ) type AuthorizationConfig struct { @@ -107,6 +110,19 @@ func (config AuthorizationConfig) New() (authorizer.Authorizer, error) { } // Keep cases in sync with constant list above. switch authorizationMode { + case modes.ModeNode: + graph := node.NewGraph() + node.AddGraphEventHandlers( + graph, + config.InformerFactory.Core().InternalVersion().Pods(), + config.InformerFactory.Core().InternalVersion().PersistentVolumes(), + ) + nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules()) + authorizers = append(authorizers, nodeAuthorizer) + + // Don't bind system:nodes to the system:node role + bootstrappolicy.AddClusterRoleBindingFilter(bootstrappolicy.OmitNodesGroupBinding) + case modes.ModeAlwaysAllow: authorizers = append(authorizers, authorizerfactory.NewAlwaysAllowAuthorizer()) case modes.ModeAlwaysDeny: diff --git a/pkg/kubeapiserver/authorizer/modes/modes.go b/pkg/kubeapiserver/authorizer/modes/modes.go index 0d2c2442090..56a708a6b1d 100644 --- a/pkg/kubeapiserver/authorizer/modes/modes.go +++ b/pkg/kubeapiserver/authorizer/modes/modes.go @@ -22,9 +22,10 @@ const ( ModeABAC string = "ABAC" ModeWebhook string = "Webhook" ModeRBAC string = "RBAC" + ModeNode string = "Node" ) -var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABAC, ModeWebhook, ModeRBAC} +var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABAC, ModeWebhook, ModeRBAC, ModeNode} // IsValidAuthorizationMode returns true if the given authorization mode is a valid one for the apiserver func IsValidAuthorizationMode(authzMode string) bool { diff --git a/plugin/pkg/auth/authorizer/BUILD b/plugin/pkg/auth/authorizer/BUILD index 7f64b63a314..48e484004c3 100644 --- a/plugin/pkg/auth/authorizer/BUILD +++ b/plugin/pkg/auth/authorizer/BUILD @@ -24,6 +24,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//plugin/pkg/auth/authorizer/node:all-srcs", "//plugin/pkg/auth/authorizer/rbac:all-srcs", ], tags = ["automanaged"], diff --git a/plugin/pkg/auth/authorizer/node/BUILD b/plugin/pkg/auth/authorizer/node/BUILD new file mode 100644 index 00000000000..8569eb1f87b --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/BUILD @@ -0,0 +1,63 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["node_authorizer_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/auth/nodeidentifier:go_default_library", + "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = [ + "graph.go", + "graph_populator.go", + "node_authorizer.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/persistentvolume:go_default_library", + "//pkg/api/pod:go_default_library", + "//pkg/apis/rbac:go_default_library", + "//pkg/auth/nodeidentifier:go_default_library", + "//pkg/client/informers/informers_generated/internalversion/core/internalversion:go_default_library", + "//plugin/pkg/auth/authorizer/rbac:go_default_library", + "//third_party/forked/gonum/graph:go_default_library", + "//third_party/forked/gonum/graph/simple:go_default_library", + "//third_party/forked/gonum/graph/traverse:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/plugin/pkg/auth/authorizer/node/OWNERS b/plugin/pkg/auth/authorizer/node/OWNERS new file mode 100644 index 00000000000..a62844dc284 --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/OWNERS @@ -0,0 +1,9 @@ +approvers: +- timstclair +- liggitt +- deads2k +reviewers: +- timstclair +- liggitt +- deads2k +- ericchiang diff --git a/plugin/pkg/auth/authorizer/node/graph.go b/plugin/pkg/auth/authorizer/node/graph.go new file mode 100644 index 00000000000..e72091f7433 --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/graph.go @@ -0,0 +1,265 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "sync" + + "k8s.io/kubernetes/pkg/api" + pvutil "k8s.io/kubernetes/pkg/api/persistentvolume" + podutil "k8s.io/kubernetes/pkg/api/pod" + "k8s.io/kubernetes/third_party/forked/gonum/graph" + "k8s.io/kubernetes/third_party/forked/gonum/graph/simple" +) + +// namedVertex implements graph.Node and remembers the type, namespace, and name of its related API object +type namedVertex struct { + name string + namespace string + id int + vertexType vertexType +} + +func newNamedVertex(vertexType vertexType, namespace, name string, id int) *namedVertex { + return &namedVertex{ + vertexType: vertexType, + name: name, + namespace: namespace, + id: id, + } +} +func (n *namedVertex) ID() int { + return n.id +} +func (n *namedVertex) String() string { + if len(n.namespace) == 0 { + return vertexTypes[n.vertexType] + ":" + n.name + } + return vertexTypes[n.vertexType] + ":" + n.namespace + "/" + n.name +} + +// destinationEdge is a graph edge that includes a denormalized reference to the final destination vertex. +// This should only be used when there is a single leaf vertex reachable from T. +type destinationEdge struct { + F graph.Node + T graph.Node + Destination graph.Node +} + +func newDestinationEdge(from, to, destination graph.Node) graph.Edge { + return &destinationEdge{F: from, T: to, Destination: destination} +} +func (e *destinationEdge) From() graph.Node { return e.F } +func (e *destinationEdge) To() graph.Node { return e.T } +func (e *destinationEdge) Weight() float64 { return 0 } +func (e *destinationEdge) DestinationID() int { return e.Destination.ID() } + +// Graph holds graph vertices and a way to look up a vertex for a particular API type/namespace/name. +// All edges point toward the vertices representing Kubernetes nodes: +// +// node <- pod +// pod <- secret,configmap,pvc +// pvc <- pv +// pv <- secret +type Graph struct { + lock sync.RWMutex + graph *simple.DirectedAcyclicGraph + // vertices is a map of type -> namespace -> name -> vertex + vertices map[vertexType]namespaceVertexMapping +} + +// namespaceVertexMapping is a map of namespace -> name -> vertex +type namespaceVertexMapping map[string]nameVertexMapping + +// nameVertexMapping is a map of name -> vertex +type nameVertexMapping map[string]*namedVertex + +func NewGraph() *Graph { + return &Graph{ + vertices: map[vertexType]namespaceVertexMapping{}, + graph: simple.NewDirectedAcyclicGraph(0, 0), + } +} + +// vertexType indicates the type of the API object the vertex represents. +// represented as a byte to minimize space used in the vertices. +type vertexType byte + +const ( + configMapVertexType vertexType = iota + nodeVertexType + podVertexType + pvcVertexType + pvVertexType + secretVertexType +) + +var vertexTypes = map[vertexType]string{ + configMapVertexType: "configmap", + nodeVertexType: "node", + podVertexType: "pod", + pvcVertexType: "pvc", + pvVertexType: "pv", + secretVertexType: "secret", +} + +// must be called under a write lock +func (g *Graph) getOrCreateVertex_locked(vertexType vertexType, namespace, name string) *namedVertex { + if vertex, exists := g.getVertex_rlocked(vertexType, namespace, name); exists { + return vertex + } + return g.createVertex_locked(vertexType, namespace, name) +} + +// must be called under a read lock +func (g *Graph) getVertex_rlocked(vertexType vertexType, namespace, name string) (*namedVertex, bool) { + vertex, exists := g.vertices[vertexType][namespace][name] + return vertex, exists +} + +// must be called under a write lock +func (g *Graph) createVertex_locked(vertexType vertexType, namespace, name string) *namedVertex { + typedVertices, exists := g.vertices[vertexType] + if !exists { + typedVertices = namespaceVertexMapping{} + g.vertices[vertexType] = typedVertices + } + + namespacedVertices, exists := typedVertices[namespace] + if !exists { + namespacedVertices = map[string]*namedVertex{} + typedVertices[namespace] = namespacedVertices + } + + vertex := newNamedVertex(vertexType, namespace, name, g.graph.NewNodeID()) + namespacedVertices[name] = vertex + g.graph.AddNode(vertex) + + return vertex +} + +// must be called under write lock +func (g *Graph) deleteVertex_locked(vertexType vertexType, namespace, name string) { + vertex, exists := g.getVertex_rlocked(vertexType, namespace, name) + if !exists { + return + } + + // find existing neighbors with a single edge (meaning we are their only neighbor) + neighborsToRemove := []graph.Node{} + g.graph.VisitFrom(vertex, func(neighbor graph.Node) bool { + // this downstream neighbor has only one edge (which must be from us), so remove them as well + if g.graph.Degree(neighbor) == 1 { + neighborsToRemove = append(neighborsToRemove, neighbor) + } + return true + }) + g.graph.VisitTo(vertex, func(neighbor graph.Node) bool { + // this upstream neighbor has only one edge (which must be to us), so remove them as well + if g.graph.Degree(neighbor) == 1 { + neighborsToRemove = append(neighborsToRemove, neighbor) + } + return true + }) + + // remove the vertex + g.graph.RemoveNode(vertex) + delete(g.vertices[vertexType][namespace], name) + if len(g.vertices[vertexType][namespace]) == 0 { + delete(g.vertices[vertexType], namespace) + } + + // remove neighbors that are now edgeless + for _, neighbor := range neighborsToRemove { + g.graph.RemoveNode(neighbor) + n := neighbor.(*namedVertex) + delete(g.vertices[n.vertexType][n.namespace], n.name) + if len(g.vertices[n.vertexType][n.namespace]) == 0 { + delete(g.vertices[n.vertexType], n.namespace) + } + } +} + +// AddPod should only be called once spec.NodeName is populated. +// It sets up edges for the following relationships (which are immutable for a pod once bound to a node): +// +// pod -> node +// +// secret -> pod +// configmap -> pod +// pvc -> pod +func (g *Graph) AddPod(pod *api.Pod) { + g.lock.Lock() + defer g.lock.Unlock() + + g.deleteVertex_locked(podVertexType, pod.Namespace, pod.Name) + podVertex := g.getOrCreateVertex_locked(podVertexType, pod.Namespace, pod.Name) + nodeVertex := g.getOrCreateVertex_locked(nodeVertexType, "", pod.Spec.NodeName) + g.graph.SetEdge(newDestinationEdge(podVertex, nodeVertex, nodeVertex)) + + podutil.VisitPodSecretNames(pod, func(secret string) bool { + g.graph.SetEdge(newDestinationEdge(g.getOrCreateVertex_locked(secretVertexType, pod.Namespace, secret), podVertex, nodeVertex)) + return true + }) + + podutil.VisitPodConfigmapNames(pod, func(configmap string) bool { + g.graph.SetEdge(newDestinationEdge(g.getOrCreateVertex_locked(configMapVertexType, pod.Namespace, configmap), podVertex, nodeVertex)) + return true + }) + + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + g.graph.SetEdge(newDestinationEdge(g.getOrCreateVertex_locked(pvcVertexType, pod.Namespace, v.PersistentVolumeClaim.ClaimName), podVertex, nodeVertex)) + } + } +} +func (g *Graph) DeletePod(name, namespace string) { + g.lock.Lock() + defer g.lock.Unlock() + g.deleteVertex_locked(podVertexType, namespace, name) +} + +// AddPV sets up edges for the following relationships: +// +// secret -> pv +// +// pv -> pvc +func (g *Graph) AddPV(pv *api.PersistentVolume) { + g.lock.Lock() + defer g.lock.Unlock() + + // clear existing edges + g.deleteVertex_locked(pvVertexType, "", pv.Name) + + // if we have a pvc, establish new edges + if pv.Spec.ClaimRef != nil { + pvVertex := g.getOrCreateVertex_locked(pvVertexType, "", pv.Name) + + // since we don't know the other end of the pvc -> pod -> node chain (or it may not even exist yet), we can't decorate these edges with kubernetes node info + g.graph.SetEdge(simple.Edge{F: pvVertex, T: g.getOrCreateVertex_locked(pvcVertexType, pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name)}) + pvutil.VisitPVSecretNames(pv, func(secret string) bool { + // This grants access to the named secret in the same namespace as the bound PVC + g.graph.SetEdge(simple.Edge{F: g.getOrCreateVertex_locked(secretVertexType, pv.Spec.ClaimRef.Namespace, secret), T: pvVertex}) + return true + }) + } +} +func (g *Graph) DeletePV(name string) { + g.lock.Lock() + defer g.lock.Unlock() + g.deleteVertex_locked(pvVertexType, "", name) +} diff --git a/plugin/pkg/auth/authorizer/node/graph_populator.go b/plugin/pkg/auth/authorizer/node/graph_populator.go new file mode 100644 index 00000000000..ecea060d6ec --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/graph_populator.go @@ -0,0 +1,108 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "github.com/golang/glog" + + "k8s.io/client-go/tools/cache" + "k8s.io/kubernetes/pkg/api" + coreinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/core/internalversion" +) + +type graphPopulator struct { + graph *Graph +} + +func AddGraphEventHandlers(graph *Graph, pods coreinformers.PodInformer, pvs coreinformers.PersistentVolumeInformer) { + g := &graphPopulator{ + graph: graph, + } + + pods.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: g.addPod, + UpdateFunc: g.updatePod, + DeleteFunc: g.deletePod, + }) + + pvs.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: g.addPV, + UpdateFunc: g.updatePV, + DeleteFunc: g.deletePV, + }) +} + +func (g *graphPopulator) addPod(obj interface{}) { + g.updatePod(nil, obj) +} + +func (g *graphPopulator) updatePod(oldObj, obj interface{}) { + pod := obj.(*api.Pod) + if len(pod.Spec.NodeName) == 0 { + // No node assigned + glog.V(5).Infof("updatePod %s/%s, no node", pod.Namespace, pod.Name) + return + } + if oldPod, ok := oldObj.(*api.Pod); ok && oldPod != nil { + if (pod.Spec.NodeName == oldPod.Spec.NodeName) && (pod.UID == oldPod.UID) { + // Node and uid are unchanged, all object references in the pod spec are immutable + glog.V(5).Infof("updatePod %s/%s, node unchanged", pod.Namespace, pod.Name) + return + } + } + glog.V(4).Infof("updatePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) + g.graph.AddPod(pod) +} + +func (g *graphPopulator) deletePod(obj interface{}) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + pod, ok := obj.(*api.Pod) + if !ok { + glog.Infof("unexpected type %T", obj) + return + } + if len(pod.Spec.NodeName) == 0 { + glog.V(5).Infof("deletePod %s/%s, no node", pod.Namespace, pod.Name) + return + } + glog.V(4).Infof("deletePod %s/%s for node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) + g.graph.DeletePod(pod.Name, pod.Namespace) +} + +func (g *graphPopulator) addPV(obj interface{}) { + g.updatePV(nil, obj) +} + +func (g *graphPopulator) updatePV(oldObj, obj interface{}) { + pv := obj.(*api.PersistentVolume) + // TODO: skip add if uid, pvc, and secrets are all identical between old and new + g.graph.AddPV(pv) +} + +func (g *graphPopulator) deletePV(obj interface{}) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + pv, ok := obj.(*api.PersistentVolume) + if !ok { + glog.Infof("unexpected type %T", obj) + return + } + g.graph.DeletePV(pv.Name) +} diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer.go b/plugin/pkg/auth/authorizer/node/node_authorizer.go new file mode 100644 index 00000000000..5e5d7362c65 --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/node_authorizer.go @@ -0,0 +1,164 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + + "github.com/golang/glog" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/kubernetes/pkg/api" + rbacapi "k8s.io/kubernetes/pkg/apis/rbac" + "k8s.io/kubernetes/pkg/auth/nodeidentifier" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" + "k8s.io/kubernetes/third_party/forked/gonum/graph" + "k8s.io/kubernetes/third_party/forked/gonum/graph/traverse" +) + +// NodeAuthorizer authorizes requests from kubelets, with the following logic: +// 1. If a request is not from a node (IdentifyNode() returns isNode=false), reject +// 2. If a specific node cannot be identified (IdentifyNode() returns nodeName=""), reject +// 3. If a request is for a secret, configmap, persistent volume or persistent volume claim, reject unless the verb is get, and the requested object is related to the requesting node: +// node <- pod +// node <- pod <- secret +// node <- pod <- configmap +// node <- pod <- pvc +// node <- pod <- pvc <- pv +// node <- pod <- pvc <- pv <- secret +// 4. For other resources, authorize all nodes uniformly using statically defined rules +type NodeAuthorizer struct { + graph *Graph + identifier nodeidentifier.NodeIdentifier + nodeRules []rbacapi.PolicyRule +} + +// New returns a new node authorizer +func NewAuthorizer(graph *Graph, identifier nodeidentifier.NodeIdentifier, rules []rbacapi.PolicyRule) authorizer.Authorizer { + return &NodeAuthorizer{ + graph: graph, + identifier: identifier, + nodeRules: rules, + } +} + +var ( + configMapResource = api.Resource("configmaps") + secretResource = api.Resource("secrets") + pvcResource = api.Resource("persistentvolumeclaims") + pvResource = api.Resource("persistentvolumes") +) + +func (r *NodeAuthorizer) Authorize(attrs authorizer.Attributes) (bool, string, error) { + nodeName, isNode := r.identifier.NodeIdentity(attrs.GetUser()) + if !isNode { + // reject requests from non-nodes + return false, "", nil + } + if len(nodeName) == 0 { + // reject requests from unidentifiable nodes + glog.V(2).Infof("NODE DENY: unknown node for user %q", attrs.GetUser().GetName()) + return false, fmt.Sprintf("unknown node for user %q", attrs.GetUser().GetName()), nil + } + + // subdivide access to specific resources + if attrs.IsResourceRequest() { + requestResource := schema.GroupResource{Group: attrs.GetAPIGroup(), Resource: attrs.GetResource()} + switch requestResource { + case secretResource: + return r.authorizeGet(nodeName, secretVertexType, attrs) + case configMapResource: + return r.authorizeGet(nodeName, configMapVertexType, attrs) + case pvcResource: + return r.authorizeGet(nodeName, pvcVertexType, attrs) + case pvResource: + return r.authorizeGet(nodeName, pvVertexType, attrs) + } + } + + // Access to other resources is not subdivided, so just evaluate against the statically defined node rules + return rbac.RulesAllow(attrs, r.nodeRules...), "", nil +} + +// authorizeGet authorizes "get" requests to objects of the specified type if they are related to the specified node +func (r *NodeAuthorizer) authorizeGet(nodeName string, startingType vertexType, attrs authorizer.Attributes) (bool, string, error) { + if attrs.GetVerb() != "get" || len(attrs.GetName()) == 0 { + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return false, "can only get individual resources of this type", nil + } + + if len(attrs.GetSubresource()) > 0 { + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return false, "cannot get subresource", nil + } + + ok, err := r.hasPathFrom(nodeName, startingType, attrs.GetNamespace(), attrs.GetName()) + if err != nil { + glog.V(2).Infof("NODE DENY: %v", err) + return false, "no path found to object", nil + } + if !ok { + glog.V(2).Infof("NODE DENY: %s %#v", nodeName, attrs) + return false, "no path found to object", nil + } + return ok, "", nil +} + +// hasPathFrom returns true if there is a directed path from the specified type/namespace/name to the specified Node +func (r *NodeAuthorizer) hasPathFrom(nodeName string, startingType vertexType, startingNamespace, startingName string) (bool, error) { + r.graph.lock.RLock() + defer r.graph.lock.RUnlock() + + nodeVertex, exists := r.graph.getVertex_rlocked(nodeVertexType, "", nodeName) + if !exists { + return false, fmt.Errorf("unknown node %s cannot get %s %s/%s", nodeName, vertexTypes[startingType], startingNamespace, startingName) + } + + startingVertex, exists := r.graph.getVertex_rlocked(startingType, startingNamespace, startingName) + if !exists { + return false, fmt.Errorf("node %s cannot get unknown %s %s/%s", nodeName, vertexTypes[startingType], startingNamespace, startingName) + } + + found := false + traversal := &traverse.VisitingDepthFirst{ + EdgeFilter: func(edge graph.Edge) bool { + if destinationEdge, ok := edge.(*destinationEdge); ok { + if destinationEdge.DestinationID() != nodeVertex.ID() { + // Don't follow edges leading to other nodes + return false + } + // We found an edge leading to the node we want + found = true + } + // Visit this edge + return true + }, + } + traversal.Walk(r.graph.graph, startingVertex, func(n graph.Node) bool { + if n.ID() == nodeVertex.ID() { + // We found the node we want + found = true + } + // Stop visiting if we've found the node we want + return found + }) + if !found { + return false, fmt.Errorf("node %s cannot get %s %s/%s, no path was found", nodeName, vertexTypes[startingType], startingNamespace, startingName) + } + return true, nil +} diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer_test.go b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go new file mode 100644 index 00000000000..39f10b62ee5 --- /dev/null +++ b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go @@ -0,0 +1,437 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + "runtime" + "runtime/pprof" + "testing" + + "os" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/auth/nodeidentifier" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" +) + +func TestAuthorizer(t *testing.T) { + g := NewGraph() + + opts := sampleDataOpts{ + nodes: 2, + namespaces: 2, + podsPerNode: 2, + sharedConfigMapsPerPod: 0, + uniqueConfigMapsPerPod: 1, + sharedSecretsPerPod: 1, + uniqueSecretsPerPod: 1, + sharedPVCsPerPod: 0, + uniquePVCsPerPod: 1, + } + pods, pvs := generate(opts) + populate(g, pods, pvs) + + identifier := nodeidentifier.NewDefaultNodeIdentifier() + authz := NewAuthorizer(g, identifier, bootstrappolicy.NodeRules()) + + node0 := &user.DefaultInfo{Name: "system:node:node0", Groups: []string{"system:nodes"}} + + tests := []struct { + name string + attrs authorizer.AttributesRecord + expect bool + }{ + { + name: "allowed configmap", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-pod0-node0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed shared secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-shared", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed shared secret via pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret-pv0-pod0-node0-ns0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumeclaims", Name: "pvc0-pod0-node0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed pv", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumes", Name: "pv0-pod0-node0-ns0", Namespace: ""}, + expect: true, + }, + + { + name: "disallowed configmap", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed shared secret via pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret-pv0-pod0-node1-ns0", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumeclaims", Name: "pvc0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed pv", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumes", Name: "pv0-pod0-node1-ns0", Namespace: ""}, + expect: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ok, _, _ := authz.Authorize(tc.attrs) + if ok != tc.expect { + t.Errorf("expected %v, got %v", tc.expect, ok) + } + }) + } +} + +func TestAuthorizerSharedResources(t *testing.T) { + g := NewGraph() + identifier := nodeidentifier.NewDefaultNodeIdentifier() + authz := NewAuthorizer(g, identifier, bootstrappolicy.NodeRules()) + + node1 := &user.DefaultInfo{Name: "system:node:node1", Groups: []string{"system:nodes"}} + node2 := &user.DefaultInfo{Name: "system:node:node2", Groups: []string{"system:nodes"}} + node3 := &user.DefaultInfo{Name: "system:node:node3", Groups: []string{"system:nodes"}} + + g.AddPod(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod1-node1", Namespace: "ns1"}, + Spec: api.PodSpec{ + NodeName: "node1", + Volumes: []api.Volume{ + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "node1-only"}}}, + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "node1-node2-only"}}}, + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "shared-all"}}}, + }, + }, + }) + g.AddPod(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod2-node2", Namespace: "ns1"}, + Spec: api.PodSpec{ + NodeName: "node2", + Volumes: []api.Volume{ + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "node1-node2-only"}}}, + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "shared-all"}}}, + }, + }, + }) + g.AddPod(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod3-node3", Namespace: "ns1"}, + Spec: api.PodSpec{ + NodeName: "node3", + Volumes: []api.Volume{ + {VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "shared-all"}}}, + }, + }, + }) + + testcases := []struct { + User user.Info + Secret string + ExpectAllowed bool + }{ + {User: node1, ExpectAllowed: true, Secret: "node1-only"}, + {User: node1, ExpectAllowed: true, Secret: "node1-node2-only"}, + {User: node1, ExpectAllowed: true, Secret: "shared-all"}, + + {User: node2, ExpectAllowed: false, Secret: "node1-only"}, + {User: node2, ExpectAllowed: true, Secret: "node1-node2-only"}, + {User: node2, ExpectAllowed: true, Secret: "shared-all"}, + + {User: node3, ExpectAllowed: false, Secret: "node1-only"}, + {User: node3, ExpectAllowed: false, Secret: "node1-node2-only"}, + {User: node3, ExpectAllowed: true, Secret: "shared-all"}, + } + + for i, tc := range testcases { + ok, _, err := authz.Authorize(authorizer.AttributesRecord{User: tc.User, ResourceRequest: true, Verb: "get", Resource: "secrets", Namespace: "ns1", Name: tc.Secret}) + if err != nil { + t.Errorf("%d: unexpected error: %v", i, err) + continue + } + if ok != tc.ExpectAllowed { + t.Errorf("%d: expected %v, got %v", i, tc.ExpectAllowed, ok) + } + } +} + +type sampleDataOpts struct { + nodes int + + namespaces int + + podsPerNode int + + sharedConfigMapsPerPod int + sharedSecretsPerPod int + sharedPVCsPerPod int + + uniqueSecretsPerPod int + uniqueConfigMapsPerPod int + uniquePVCsPerPod int +} + +func BenchmarkPopulationAllocation(b *testing.B) { + opts := sampleDataOpts{ + nodes: 500, + namespaces: 200, + podsPerNode: 200, + sharedConfigMapsPerPod: 0, + uniqueConfigMapsPerPod: 1, + sharedSecretsPerPod: 1, + uniqueSecretsPerPod: 1, + sharedPVCsPerPod: 0, + uniquePVCsPerPod: 1, + } + + pods, pvs := generate(opts) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + g := NewGraph() + populate(g, pods, pvs) + } +} + +func BenchmarkPopulationRetention(b *testing.B) { + + // Run with: + // go test ./plugin/pkg/auth/authorizer/node -benchmem -bench . -run None -v -o node.test -timeout 300m + + // Evaluate retained memory with: + // go tool pprof --inuse_space node.test plugin/pkg/auth/authorizer/node/BenchmarkPopulationRetention.profile + // list populate + + opts := sampleDataOpts{ + nodes: 500, + namespaces: 200, + podsPerNode: 200, + sharedConfigMapsPerPod: 0, + uniqueConfigMapsPerPod: 1, + sharedSecretsPerPod: 1, + uniqueSecretsPerPod: 1, + sharedPVCsPerPod: 0, + uniquePVCsPerPod: 1, + } + + pods, pvs := generate(opts) + // Garbage collect before the first iteration + runtime.GC() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + g := NewGraph() + populate(g, pods, pvs) + + if i == 0 { + f, _ := os.Create("BenchmarkPopulationRetention.profile") + runtime.GC() + pprof.WriteHeapProfile(f) + f.Close() + // reference the graph to keep it from getting garbage collected + _ = fmt.Sprintf("%T\n", g) + } + } +} + +func BenchmarkAuthorization(b *testing.B) { + g := NewGraph() + + opts := sampleDataOpts{ + nodes: 500, + namespaces: 200, + podsPerNode: 200, + sharedConfigMapsPerPod: 0, + uniqueConfigMapsPerPod: 1, + sharedSecretsPerPod: 1, + uniqueSecretsPerPod: 1, + sharedPVCsPerPod: 0, + uniquePVCsPerPod: 1, + } + pods, pvs := generate(opts) + populate(g, pods, pvs) + + identifier := nodeidentifier.NewDefaultNodeIdentifier() + authz := NewAuthorizer(g, identifier, bootstrappolicy.NodeRules()) + + node0 := &user.DefaultInfo{Name: "system:node:node0", Groups: []string{"system:nodes"}} + + tests := []struct { + name string + attrs authorizer.AttributesRecord + expect bool + }{ + { + name: "allowed configmap", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-pod0-node0", Namespace: "ns0"}, + expect: true, + }, + { + name: "allowed shared secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-shared", Namespace: "ns0"}, + expect: true, + }, + { + name: "disallowed configmap", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "configmaps", Name: "configmap0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed secret via pod", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed shared secret via pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "secrets", Name: "secret-pv0-pod0-node1-ns0", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed pvc", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumeclaims", Name: "pvc0-pod0-node1", Namespace: "ns0"}, + expect: false, + }, + { + name: "disallowed pv", + attrs: authorizer.AttributesRecord{User: node0, ResourceRequest: true, Verb: "get", Resource: "persistentvolumes", Name: "pv0-pod0-node1-ns0", Namespace: ""}, + expect: false, + }, + } + + b.ResetTimer() + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + ok, _, _ := authz.Authorize(tc.attrs) + if ok != tc.expect { + b.Errorf("expected %v, got %v", tc.expect, ok) + } + } + }) + } +} + +func populate(graph *Graph, pods []*api.Pod, pvs []*api.PersistentVolume) { + p := &graphPopulator{} + p.graph = graph + for _, pod := range pods { + p.addPod(pod) + } + for _, pv := range pvs { + p.addPV(pv) + } +} + +// generate creates sample pods and persistent volumes based on the provided options. +// the secret/configmap/pvc/node references in the pod and pv objects are named to indicate the connections between the objects. +// for example, secret0-pod0-node0 is a secret referenced by pod0 which is bound to node0. +// when populated into the graph, the node authorizer should allow node0 to access that secret, but not node1. +func generate(opts sampleDataOpts) ([]*api.Pod, []*api.PersistentVolume) { + pods := make([]*api.Pod, 0, opts.nodes*opts.podsPerNode) + pvs := make([]*api.PersistentVolume, 0, (opts.nodes*opts.podsPerNode*opts.uniquePVCsPerPod)+(opts.sharedPVCsPerPod*opts.namespaces)) + + for n := 0; n < opts.nodes; n++ { + nodeName := fmt.Sprintf("node%d", n) + for p := 0; p < opts.podsPerNode; p++ { + pod := &api.Pod{} + pod.Namespace = fmt.Sprintf("ns%d", p%opts.namespaces) + pod.Name = fmt.Sprintf("pod%d-%s", p, nodeName) + pod.Spec.NodeName = nodeName + + for i := 0; i < opts.uniqueSecretsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + Secret: &api.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-%s", i, pod.Name)}, + }}) + } + for i := 0; i < opts.sharedSecretsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + Secret: &api.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-shared", i)}, + }}) + } + + for i := 0; i < opts.uniqueConfigMapsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + ConfigMap: &api.ConfigMapVolumeSource{LocalObjectReference: api.LocalObjectReference{Name: fmt.Sprintf("configmap%d-%s", i, pod.Name)}}, + }}) + } + for i := 0; i < opts.sharedConfigMapsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + ConfigMap: &api.ConfigMapVolumeSource{LocalObjectReference: api.LocalObjectReference{Name: fmt.Sprintf("configmap%d-shared", i)}}, + }}) + } + + for i := 0; i < opts.uniquePVCsPerPod; i++ { + pv := &api.PersistentVolume{} + pv.Name = fmt.Sprintf("pv%d-%s-%s", i, pod.Name, pod.Namespace) + pv.Spec.FlexVolume = &api.FlexVolumeSource{SecretRef: &api.LocalObjectReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} + pv.Spec.ClaimRef = &api.ObjectReference{Name: fmt.Sprintf("pvc%d-%s", i, pod.Name), Namespace: pod.Namespace} + pvs = append(pvs, pv) + + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, + }}) + } + for i := 0; i < opts.sharedPVCsPerPod; i++ { + pv := &api.PersistentVolume{} + pv.Name = fmt.Sprintf("pv%d-shared-%s", i, pod.Namespace) + pv.Spec.FlexVolume = &api.FlexVolumeSource{SecretRef: &api.LocalObjectReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} + pv.Spec.ClaimRef = &api.ObjectReference{Name: fmt.Sprintf("pvc%d-shared", i), Namespace: pod.Namespace} + pvs = append(pvs, pv) + + pod.Spec.Volumes = append(pod.Spec.Volumes, api.Volume{VolumeSource: api.VolumeSource{ + PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, + }}) + } + + pods = append(pods, pod) + } + } + return pods, pvs +} diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 4e4e3a8e09c..1697ad680d9 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -91,37 +91,37 @@ func NodeRules() []rbac.PolicyRule { // Needed to check API access. These creates are non-mutating rbac.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(), rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews", "localsubjectaccessreviews").RuleOrDie(), + // Needed to build serviceLister, to populate env vars for services rbac.NewRule(Read...).Groups(legacyGroup).Resources("services").RuleOrDie(), - // Nodes can register themselves - // TODO: restrict to creating a node with the same name they announce + + // Nodes can register Node API objects and report status. + // Use the NodeRestriction admission plugin to limit a node to creating/updating its own API object. rbac.NewRule("create", "get", "list", "watch").Groups(legacyGroup).Resources("nodes").RuleOrDie(), - // TODO: restrict to the bound node once supported rbac.NewRule("update", "patch").Groups(legacyGroup).Resources("nodes/status").RuleOrDie(), rbac.NewRule("update", "patch", "delete").Groups(legacyGroup).Resources("nodes").RuleOrDie(), - // TODO: restrict to the bound node as creator once supported + // TODO: restrict to the bound node as creator in the NodeRestrictions admission plugin rbac.NewRule("create", "update", "patch").Groups(legacyGroup).Resources("events").RuleOrDie(), - // TODO: restrict to pods scheduled on the bound node once supported + // TODO: restrict to pods scheduled on the bound node once field selectors are supported by list/watch authorization rbac.NewRule(Read...).Groups(legacyGroup).Resources("pods").RuleOrDie(), - // TODO: remove once mirror pods are removed - // TODO: restrict deletion to mirror pods created by the bound node once supported - // Needed for the node to create/delete mirror pods + // Needed for the node to create/delete mirror pods. + // Use the NodeRestriction admission plugin to limit a node to creating/deleting mirror pods bound to itself. rbac.NewRule("create", "delete").Groups(legacyGroup).Resources("pods").RuleOrDie(), - // TODO: restrict to pods scheduled on the bound node once supported + // Needed for the node to report status of pods it is running. + // Use the NodeRestriction admission plugin to limit a node to updating status of pods bound to itself. rbac.NewRule("update").Groups(legacyGroup).Resources("pods/status").RuleOrDie(), - // TODO: restrict to secrets and configmaps used by pods scheduled on bound node once supported // Needed for imagepullsecrets, rbd/ceph and secret volumes, and secrets in envs // Needed for configmap volume and envs + // Use the NodeRestriction admission plugin to limit a node to get secrets/configmaps referenced by pods bound to itself. rbac.NewRule("get").Groups(legacyGroup).Resources("secrets", "configmaps").RuleOrDie(), - // TODO: restrict to claims/volumes used by pods scheduled on bound node once supported // Needed for persistent volumes + // Use the NodeRestriction admission plugin to limit a node to get pv/pvc objects referenced by pods bound to itself. rbac.NewRule("get").Groups(legacyGroup).Resources("persistentvolumeclaims", "persistentvolumes").RuleOrDie(), - // TODO: restrict to namespaces of pods scheduled on bound node once supported - // TODO: change glusterfs to use DNS lookup so this isn't needed? + // TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node // Needed for glusterfs volumes rbac.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), // Used to create a certificatesigningrequest for a node-specific client certificate, and watch @@ -362,18 +362,70 @@ func ClusterRoles() []rbac.ClusterRole { return roles } +// ClusterRoleBindingFilter can modify and return or omit (by returning nil) a role binding +type ClusterRoleBindingFilter func(*rbac.ClusterRoleBinding) *rbac.ClusterRoleBinding + +// AddClusterRoleBindingFilter adds the given filter to the list that is invoked when determing bootstrap roles to reconcile. +func AddClusterRoleBindingFilter(filter ClusterRoleBindingFilter) { + clusterRoleBindingFilters = append(clusterRoleBindingFilters, filter) +} + +// ClearClusterRoleBindingFilters removes any filters added using AddClusterRoleBindingFilter +func ClearClusterRoleBindingFilters() { + clusterRoleBindingFilters = nil +} + +const systemNodeRoleName = "system:node" + +var clusterRoleBindingFilters []ClusterRoleBindingFilter + +// OmitNodesGroupBinding is a filter that omits the deprecated binding for the system:nodes group to the system:node role. +var OmitNodesGroupBinding = ClusterRoleBindingFilter(func(binding *rbac.ClusterRoleBinding) *rbac.ClusterRoleBinding { + if binding.RoleRef.Name == systemNodeRoleName { + subjects := []rbac.Subject{} + for _, subject := range binding.Subjects { + if subject.Kind == rbac.GroupKind && subject.Name == user.NodesGroup { + continue + } + subjects = append(subjects, subject) + } + binding.Subjects = subjects + } + return binding +}) + // ClusterRoleBindings return default rolebindings to the default roles func ClusterRoleBindings() []rbac.ClusterRoleBinding { rolebindings := []rbac.ClusterRoleBinding{ rbac.NewClusterBinding("cluster-admin").Groups(user.SystemPrivilegedGroup).BindingOrDie(), rbac.NewClusterBinding("system:discovery").Groups(user.AllAuthenticated, user.AllUnauthenticated).BindingOrDie(), rbac.NewClusterBinding("system:basic-user").Groups(user.AllAuthenticated, user.AllUnauthenticated).BindingOrDie(), - rbac.NewClusterBinding("system:node").Groups(user.NodesGroup).BindingOrDie(), rbac.NewClusterBinding("system:node-proxier").Users(user.KubeProxy).BindingOrDie(), rbac.NewClusterBinding("system:kube-controller-manager").Users(user.KubeControllerManager).BindingOrDie(), rbac.NewClusterBinding("system:kube-dns").SAs("kube-system", "kube-dns").BindingOrDie(), rbac.NewClusterBinding("system:kube-scheduler").Users(user.KubeScheduler).BindingOrDie(), + + // This default system:nodes binding is deprecated in 1.7 with the availability of the Node authorizer. + // If an admin wants to grant the system:node role (which cannot partition Node API access), they will need to create their own clusterrolebinding. + // TODO: Remove the subjects from this binding in 1.8 (leave the empty binding for tightening reconciliation), and remove AddClusterRoleBindingFilter() + rbac.NewClusterBinding(systemNodeRoleName).Groups(user.NodesGroup).BindingOrDie(), } + addClusterRoleBindingLabel(rolebindings) - return rolebindings + + retval := []rbac.ClusterRoleBinding{} + for i := range rolebindings { + binding := &rolebindings[i] + for _, filter := range clusterRoleBindingFilters { + binding = filter(binding) + if binding == nil { + break + } + } + if binding != nil { + retval = append(retval, *binding) + } + } + + return retval } diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 166af52f746..c7d2f818129 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -12,6 +12,7 @@ go_test( srcs = [ "accessreview_test.go", "auth_test.go", + "node_test.go", "rbac_test.go", ], tags = [ @@ -27,7 +28,10 @@ go_test( "//pkg/apis/extensions:go_default_library", "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", + "//pkg/auth/nodeidentifier:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", + "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/kubeapiserver/authorizer:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/rbac/clusterrole:go_default_library", "//pkg/registry/rbac/clusterrole/storage:go_default_library", @@ -38,10 +42,14 @@ go_test( "//pkg/registry/rbac/rolebinding:go_default_library", "//pkg/registry/rbac/rolebinding/storage:go_default_library", "//plugin/pkg/admission/admit:go_default_library", + "//plugin/pkg/admission/noderestriction:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", + "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go new file mode 100644 index 00000000000..d8d9d895fac --- /dev/null +++ b/test/integration/auth/node_test.go @@ -0,0 +1,303 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/auth/nodeidentifier" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + "k8s.io/kubernetes/pkg/kubeapiserver/authorizer" + "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestNodeAuthorizer(t *testing.T) { + // Start the server so we know the address + h := &framework.MasterHolder{Initialized: make(chan struct{})} + apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + <-h.Initialized + h.M.GenericAPIServer.Handler.ServeHTTP(w, req) + })) + + // Build client config, clientset, and informers + clientConfig := &restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}} + superuserClient := clientsetForUser("admin/system:masters", clientConfig) + informerFactory := informers.NewSharedInformerFactory(superuserClient, time.Minute) + + // Set up Node+RBAC authorizer + authorizerConfig := &authorizer.AuthorizationConfig{ + AuthorizationModes: []string{"Node", "RBAC"}, + InformerFactory: informerFactory, + } + nodeRBACAuthorizer, err := authorizerConfig.New() + if err != nil { + t.Fatal(err) + } + defer bootstrappolicy.ClearClusterRoleBindingFilters() + + // Set up NodeRestriction admission + nodeRestrictionAdmission := noderestriction.NewPlugin(nodeidentifier.NewDefaultNodeIdentifier()) + nodeRestrictionAdmission.SetInternalKubeClientSet(superuserClient) + if err := nodeRestrictionAdmission.Validate(); err != nil { + t.Fatal(err) + } + + // Start the server + masterConfig := framework.NewIntegrationTestMasterConfig() + masterConfig.GenericConfig.Authenticator = newFakeAuthenticator() + masterConfig.GenericConfig.Authorizer = nodeRBACAuthorizer + masterConfig.GenericConfig.AdmissionControl = nodeRestrictionAdmission + _, _, closeFn := framework.RunAMasterUsingServer(masterConfig, apiServer, h) + defer closeFn() + + // Start the informers + stopCh := make(chan struct{}) + defer close(stopCh) + informerFactory.Start(stopCh) + + // Wait for a healthy server + for { + result := superuserClient.Core().RESTClient().Get().AbsPath("/healthz").Do() + _, err := result.Raw() + if err == nil { + break + } + t.Log(err) + time.Sleep(time.Second) + } + + // Create objects + if _, err := superuserClient.Core().Secrets("ns").Create(&api.Secret{ObjectMeta: metav1.ObjectMeta{Name: "mysecret"}}); err != nil { + t.Fatal(err) + } + if _, err := superuserClient.Core().Secrets("ns").Create(&api.Secret{ObjectMeta: metav1.ObjectMeta{Name: "mypvsecret"}}); err != nil { + t.Fatal(err) + } + if _, err := superuserClient.Core().ConfigMaps("ns").Create(&api.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "myconfigmap"}}); err != nil { + t.Fatal(err) + } + if _, err := superuserClient.Core().PersistentVolumeClaims("ns").Create(&api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "mypvc"}, + Spec: api.PersistentVolumeClaimSpec{ + AccessModes: []api.PersistentVolumeAccessMode{api.ReadOnlyMany}, + Resources: api.ResourceRequirements{Requests: api.ResourceList{api.ResourceStorage: resource.MustParse("1")}}, + }, + }); err != nil { + t.Fatal(err) + } + if _, err := superuserClient.Core().PersistentVolumes().Create(&api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "mypv"}, + Spec: api.PersistentVolumeSpec{ + AccessModes: []api.PersistentVolumeAccessMode{api.ReadOnlyMany}, + Capacity: api.ResourceList{api.ResourceStorage: resource.MustParse("1")}, + ClaimRef: &api.ObjectReference{Namespace: "ns", Name: "mypvc"}, + PersistentVolumeSource: api.PersistentVolumeSource{AzureFile: &api.AzureFileVolumeSource{ShareName: "default", SecretName: "mypvsecret"}}, + }, + }); err != nil { + t.Fatal(err) + } + + getSecret := func(client clientset.Interface) error { + _, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{}) + return err + } + getPVSecret := func(client clientset.Interface) error { + _, err := client.Core().Secrets("ns").Get("mypvsecret", metav1.GetOptions{}) + return err + } + getConfigMap := func(client clientset.Interface) error { + _, err := client.Core().ConfigMaps("ns").Get("myconfigmap", metav1.GetOptions{}) + return err + } + getPVC := func(client clientset.Interface) error { + _, err := client.Core().PersistentVolumeClaims("ns").Get("mypvc", metav1.GetOptions{}) + return err + } + getPV := func(client clientset.Interface) error { + _, err := client.Core().PersistentVolumes().Get("mypv", metav1.GetOptions{}) + return err + } + + createNode2NormalPod := func(client clientset.Interface) error { + _, err := client.Core().Pods("ns").Create(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "node2normalpod"}, + Spec: api.PodSpec{ + NodeName: "node2", + Containers: []api.Container{{Name: "image", Image: "busybox"}}, + Volumes: []api.Volume{ + {Name: "secret", VolumeSource: api.VolumeSource{Secret: &api.SecretVolumeSource{SecretName: "mysecret"}}}, + {Name: "cm", VolumeSource: api.VolumeSource{ConfigMap: &api.ConfigMapVolumeSource{LocalObjectReference: api.LocalObjectReference{Name: "myconfigmap"}}}}, + {Name: "pvc", VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"}}}, + }, + }, + }) + return err + } + updateNode2NormalPodStatus := func(client clientset.Interface) error { + startTime := metav1.NewTime(time.Now()) + _, err := client.Core().Pods("ns").UpdateStatus(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "node2normalpod"}, + Status: api.PodStatus{StartTime: &startTime}, + }) + return err + } + deleteNode2NormalPod := func(client clientset.Interface) error { + zero := int64(0) + return client.Core().Pods("ns").Delete("node2normalpod", &metav1.DeleteOptions{GracePeriodSeconds: &zero}) + } + + createNode2MirrorPod := func(client clientset.Interface) error { + _, err := client.Core().Pods("ns").Create(&api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2mirrorpod", + Annotations: map[string]string{api.MirrorPodAnnotationKey: "true"}, + }, + Spec: api.PodSpec{ + NodeName: "node2", + Containers: []api.Container{{Name: "image", Image: "busybox"}}, + }, + }) + return err + } + deleteNode2MirrorPod := func(client clientset.Interface) error { + zero := int64(0) + return client.Core().Pods("ns").Delete("node2mirrorpod", &metav1.DeleteOptions{GracePeriodSeconds: &zero}) + } + + createNode2 := func(client clientset.Interface) error { + _, err := client.Core().Nodes().Create(&api.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}}) + return err + } + updateNode2Status := func(client clientset.Interface) error { + _, err := client.Core().Nodes().UpdateStatus(&api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node2"}, + Status: api.NodeStatus{}, + }) + return err + } + deleteNode2 := func(client clientset.Interface) error { + return client.Core().Nodes().Delete("node2", nil) + } + + nodeanonClient := clientsetForUser("unknown/system:nodes", clientConfig) + node1Client := clientsetForUser("system:node:node1/system:nodes", clientConfig) + node2Client := clientsetForUser("system:node:node2/system:nodes", clientConfig) + + // all node requests from node1 and unknown node fail + expectForbidden(t, getSecret(nodeanonClient)) + expectForbidden(t, getPVSecret(nodeanonClient)) + expectForbidden(t, getConfigMap(nodeanonClient)) + expectForbidden(t, getPVC(nodeanonClient)) + expectForbidden(t, getPV(nodeanonClient)) + expectForbidden(t, createNode2NormalPod(nodeanonClient)) + expectForbidden(t, createNode2MirrorPod(nodeanonClient)) + expectForbidden(t, deleteNode2MirrorPod(nodeanonClient)) + expectForbidden(t, createNode2(nodeanonClient)) + expectForbidden(t, updateNode2Status(nodeanonClient)) + expectForbidden(t, deleteNode2(nodeanonClient)) + + expectForbidden(t, getSecret(node1Client)) + expectForbidden(t, getPVSecret(node1Client)) + expectForbidden(t, getConfigMap(node1Client)) + expectForbidden(t, getPVC(node1Client)) + expectForbidden(t, getPV(node1Client)) + expectForbidden(t, createNode2NormalPod(nodeanonClient)) + expectForbidden(t, createNode2MirrorPod(node1Client)) + expectForbidden(t, deleteNode2MirrorPod(node1Client)) + expectForbidden(t, createNode2(node1Client)) + expectForbidden(t, updateNode2Status(node1Client)) + expectForbidden(t, deleteNode2(node1Client)) + + // related object requests from node2 fail + expectForbidden(t, getSecret(node2Client)) + expectForbidden(t, getPVSecret(node2Client)) + expectForbidden(t, getConfigMap(node2Client)) + expectForbidden(t, getPVC(node2Client)) + expectForbidden(t, getPV(node2Client)) + expectForbidden(t, createNode2NormalPod(nodeanonClient)) + // mirror pod and self node lifecycle is allowed + expectAllowed(t, createNode2MirrorPod(node2Client)) + expectAllowed(t, deleteNode2MirrorPod(node2Client)) + expectAllowed(t, createNode2(node2Client)) + expectAllowed(t, updateNode2Status(node2Client)) + expectAllowed(t, deleteNode2(node2Client)) + + // create a pod as an admin to add object references + expectAllowed(t, createNode2NormalPod(superuserClient)) + + // unidentifiable node and node1 are still forbidden + expectForbidden(t, getSecret(nodeanonClient)) + expectForbidden(t, getPVSecret(nodeanonClient)) + expectForbidden(t, getConfigMap(nodeanonClient)) + expectForbidden(t, getPVC(nodeanonClient)) + expectForbidden(t, getPV(nodeanonClient)) + expectForbidden(t, createNode2NormalPod(nodeanonClient)) + expectForbidden(t, updateNode2NormalPodStatus(nodeanonClient)) + expectForbidden(t, deleteNode2NormalPod(nodeanonClient)) + expectForbidden(t, createNode2MirrorPod(nodeanonClient)) + expectForbidden(t, deleteNode2MirrorPod(nodeanonClient)) + + expectForbidden(t, getSecret(node1Client)) + expectForbidden(t, getPVSecret(node1Client)) + expectForbidden(t, getConfigMap(node1Client)) + expectForbidden(t, getPVC(node1Client)) + expectForbidden(t, getPV(node1Client)) + expectForbidden(t, createNode2NormalPod(node1Client)) + expectForbidden(t, updateNode2NormalPodStatus(node1Client)) + expectForbidden(t, deleteNode2NormalPod(node1Client)) + expectForbidden(t, createNode2MirrorPod(node1Client)) + expectForbidden(t, deleteNode2MirrorPod(node1Client)) + + // node2 can get referenced objects now + expectAllowed(t, getSecret(node2Client)) + expectAllowed(t, getPVSecret(node2Client)) + expectAllowed(t, getConfigMap(node2Client)) + expectAllowed(t, getPVC(node2Client)) + expectAllowed(t, getPV(node2Client)) + expectForbidden(t, createNode2NormalPod(node2Client)) + expectAllowed(t, updateNode2NormalPodStatus(node2Client)) + expectAllowed(t, deleteNode2NormalPod(node2Client)) + expectAllowed(t, createNode2MirrorPod(node2Client)) + expectAllowed(t, deleteNode2MirrorPod(node2Client)) +} + +func expectForbidden(t *testing.T, err error) { + if !errors.IsForbidden(err) { + _, file, line, _ := runtime.Caller(1) + t.Errorf("%s:%d: Expected forbidden error, got %v", filepath.Base(file), line, err) + } +} + +func expectAllowed(t *testing.T, err error) { + if err != nil { + _, file, line, _ := runtime.Caller(1) + t.Errorf("%s:%d: Expected no error, got %v", filepath.Base(file), line, err) + } +}