diff --git a/docs/config.md b/docs/config.md index 1aacf15f1..e9a32435a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -35,6 +35,10 @@ version = 2 # enable_selinux indicates to enable the selinux support. enable_selinux = false + # selinux_category_range allows the upper bound on the category range to be set. + # if not specified or set to 0, defaults to 1024 from the selinux package. + selinux_category_range = 1024 + # sandbox_image is the image used by sandbox container. sandbox_image = "k8s.gcr.io/pause:3.2" diff --git a/pkg/config/config.go b/pkg/config/config.go index 579d72480..7cfb6d51f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -198,6 +198,9 @@ type PluginConfig struct { StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"` // EnableSelinux indicates to enable the selinux support. EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux"` + // SelinuxCategoryRange allows the upper bound on the category range to be set. + // If not specified or set to 0, defaults to 1024 from the selinux package. + SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"` // SandboxImage is the image used by sandbox container. SandboxImage string `toml:"sandbox_image" json:"sandboxImage"` // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. diff --git a/pkg/config/config_unix.go b/pkg/config/config_unix.go index 906301726..9df456b53 100644 --- a/pkg/config/config_unix.go +++ b/pkg/config/config_unix.go @@ -44,12 +44,13 @@ func DefaultConfig() PluginConfig { }, }, }, - DisableTCPService: true, - StreamServerAddress: "127.0.0.1", - StreamServerPort: "0", - StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour - EnableSelinux: false, - EnableTLSStreaming: false, + DisableTCPService: true, + StreamServerAddress: "127.0.0.1", + StreamServerPort: "0", + StreamIdleTimeout: streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour + EnableSelinux: false, + SelinuxCategoryRange: 1024, + EnableTLSStreaming: false, X509KeyPairStreaming: X509KeyPairStreaming{ TLSKeyFile: "", TLSCertFile: "", diff --git a/pkg/server/service_unix.go b/pkg/server/service_unix.go index 44f43f8c4..a1d9c9038 100644 --- a/pkg/server/service_unix.go +++ b/pkg/server/service_unix.go @@ -44,6 +44,9 @@ func (c *criService) initPlatform() error { if !selinux.GetEnabled() { logrus.Warn("Selinux is not supported") } + if r := c.config.SelinuxCategoryRange; r > 0 { + selinux.CategoryRange = uint32(r) + } } else { selinux.SetDisabled() } diff --git a/vendor.conf b/vendor.conf index 81955b62f..df5449187 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,7 +1,8 @@ # cri dependencies github.com/docker/docker 4634ce647cf2ce2c6031129ccd109e557244986f -github.com/opencontainers/selinux v1.5.1 +github.com/opencontainers/selinux v1.6.0 github.com/tchap/go-patricia v2.2.6 +github.com/willf/bitset d5bec3311243426a3c6d1b7a795f24b17c686dbb # 1.1.10+ used by selinux pkg # containerd dependencies github.com/beorn7/perks v1.0.1 diff --git a/vendor/github.com/opencontainers/selinux/README.md b/vendor/github.com/opencontainers/selinux/README.md index 04594bd4e..a9f0cc183 100644 --- a/vendor/github.com/opencontainers/selinux/README.md +++ b/vendor/github.com/opencontainers/selinux/README.md @@ -4,7 +4,11 @@ Common SELinux package used across the container ecosystem. -Please see the [godoc](https://godoc.org/github.com/opencontainers/selinux) for more information. +## Usage + +When compiling consumers of this project, the `selinux` build tag must be used to enable selinux functionality. + +For complete documentation, see [godoc](https://godoc.org/github.com/opencontainers/selinux). ## Code of Conduct diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/doc.go b/vendor/github.com/opencontainers/selinux/go-selinux/doc.go new file mode 100644 index 000000000..79a8e6446 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/doc.go @@ -0,0 +1,21 @@ +/* +Package selinux provides a high-level interface for interacting with selinux. + +This package uses a selinux build tag to enable the selinux functionality. This +allows non-linux and linux users who do not have selinux support to still use +tools that rely on this library. + +To compile with full selinux support use the -tags=selinux option in your build +and test commands. + +Usage: + + import "github.com/opencontainers/selinux/go-selinux" + + // Ensure that selinux is enforcing mode. + if selinux.EnforceMode() != selinux.Enforcing { + selinux.SetEnforceMode(selinux.Enforcing) + } + +*/ +package selinux diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/label/label.go b/vendor/github.com/opencontainers/selinux/go-selinux/label/label.go index 6e38d3d32..fea096c18 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/label/label.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/label/label.go @@ -1,6 +1,8 @@ package label import ( + "fmt" + "github.com/opencontainers/selinux/go-selinux" ) @@ -46,7 +48,7 @@ var PidLabel = selinux.PidLabel // Init initialises the labeling system func Init() { - selinux.GetEnabled() + _ = selinux.GetEnabled() } // ClearLabels will clear all reserved labels @@ -75,3 +77,21 @@ func ReleaseLabel(label string) error { // can be used to set duplicate labels on future container processes // Deprecated: use selinux.DupSecOpt var DupSecOpt = selinux.DupSecOpt + +// FormatMountLabel returns a string to be used by the mount command. +// The format of this string will be used to alter the labeling of the mountpoint. +// The string returned is suitable to be used as the options field of the mount command. +// If you need to have additional mount point options, you can pass them in as +// the first parameter. Second parameter is the label that you wish to apply +// to all content in the mount point. +func FormatMountLabel(src, mountLabel string) string { + if mountLabel != "" { + switch src { + case "": + src = fmt.Sprintf("context=%q", mountLabel) + default: + src = fmt.Sprintf("%s,context=%q", src, mountLabel) + } + } + return src +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_selinux.go b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_selinux.go index 903829958..10ac15a85 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_selinux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_selinux.go @@ -3,7 +3,6 @@ package label import ( - "fmt" "os" "os/user" "strings" @@ -43,7 +42,7 @@ func InitLabels(options []string) (plabel string, mlabel string, Err error) { if err != nil { return "", "", err } - + mcsLevel := pcon["level"] mcon, err := selinux.NewContext(mountLabel) if err != nil { return "", "", err @@ -62,16 +61,21 @@ func InitLabels(options []string) (plabel string, mlabel string, Err error) { } if con[0] == "filetype" { mcon["type"] = con[1] + continue } pcon[con[0]] = con[1] if con[0] == "level" || con[0] == "user" { mcon[con[0]] = con[1] } } - selinux.ReleaseLabel(processLabel) - processLabel = pcon.Get() + if pcon.Get() != processLabel { + if pcon["level"] != mcsLevel { + selinux.ReleaseLabel(processLabel) + } + processLabel = pcon.Get() + selinux.ReserveLabel(processLabel) + } mountLabel = mcon.Get() - selinux.ReserveLabel(processLabel) } return processLabel, mountLabel, nil } @@ -82,24 +86,6 @@ func GenLabels(options string) (string, string, error) { return InitLabels(strings.Fields(options)) } -// FormatMountLabel returns a string to be used by the mount command. -// The format of this string will be used to alter the labeling of the mountpoint. -// The string returned is suitable to be used as the options field of the mount command. -// If you need to have additional mount point options, you can pass them in as -// the first parameter. Second parameter is the label that you wish to apply -// to all content in the mount point. -func FormatMountLabel(src, mountLabel string) string { - if mountLabel != "" { - switch src { - case "": - src = fmt.Sprintf("context=%q", mountLabel) - default: - src = fmt.Sprintf("%s,context=%q", src, mountLabel) - } - } - return src -} - // SetFileLabel modifies the "path" label to the specified file label func SetFileLabel(path string, fileLabel string) error { if !selinux.GetEnabled() || fileLabel == "" { diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_stub.go b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_stub.go index cda59d671..a7d2d5e34 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/label/label_stub.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/label/label_stub.go @@ -15,10 +15,6 @@ func GenLabels(options string) (string, string, error) { return "", "", nil } -func FormatMountLabel(src string, mountLabel string) string { - return src -} - func SetFileLabel(path string, fileLabel string) error { return nil } @@ -34,7 +30,6 @@ func Relabel(path string, fileLabel string, shared bool) error { // DisableSecOpt returns a security opt that can disable labeling // support for future container processes func DisableSecOpt() []string { - // TODO the selinux.DisableSecOpt stub returns []string{"disable"} instead of "nil" return nil } diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go new file mode 100644 index 000000000..50760dc93 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux.go @@ -0,0 +1,249 @@ +package selinux + +import ( + "github.com/pkg/errors" +) + +const ( + // Enforcing constant indicate SELinux is in enforcing mode + Enforcing = 1 + // Permissive constant to indicate SELinux is in permissive mode + Permissive = 0 + // Disabled constant to indicate SELinux is disabled + Disabled = -1 + + // DefaultCategoryRange is the upper bound on the category range + DefaultCategoryRange = uint32(1024) +) + +var ( + // ErrMCSAlreadyExists is returned when trying to allocate a duplicate MCS. + ErrMCSAlreadyExists = errors.New("MCS label already exists") + // ErrEmptyPath is returned when an empty path has been specified. + ErrEmptyPath = errors.New("empty path") + + // InvalidLabel is returned when an invalid label is specified. + InvalidLabel = errors.New("Invalid Label") + + // ErrIncomparable is returned two levels are not comparable + ErrIncomparable = errors.New("incomparable levels") + // ErrLevelSyntax is returned when a sensitivity or category do not have correct syntax in a level + ErrLevelSyntax = errors.New("invalid level syntax") + + // CategoryRange allows the upper bound on the category range to be adjusted + CategoryRange = DefaultCategoryRange +) + +// Context is a representation of the SELinux label broken into 4 parts +type Context map[string]string + +// SetDisabled disables SELinux support for the package +func SetDisabled() { + setDisabled() +} + +// GetEnabled returns whether SELinux is currently enabled. +func GetEnabled() bool { + return getEnabled() +} + +// ClassIndex returns the int index for an object class in the loaded policy, +// or -1 and an error +func ClassIndex(class string) (int, error) { + return classIndex(class) +} + +// SetFileLabel sets the SELinux label for this path or returns an error. +func SetFileLabel(fpath string, label string) error { + return setFileLabel(fpath, label) +} + +// FileLabel returns the SELinux label for this path or returns an error. +func FileLabel(fpath string) (string, error) { + return fileLabel(fpath) +} + +// SetFSCreateLabel tells kernel the label to create all file system objects +// created by this task. Setting label="" to return to default. +func SetFSCreateLabel(label string) error { + return setFSCreateLabel(label) +} + +// FSCreateLabel returns the default label the kernel which the kernel is using +// for file system objects created by this task. "" indicates default. +func FSCreateLabel() (string, error) { + return fsCreateLabel() +} + +// CurrentLabel returns the SELinux label of the current process thread, or an error. +func CurrentLabel() (string, error) { + return currentLabel() +} + +// PidLabel returns the SELinux label of the given pid, or an error. +func PidLabel(pid int) (string, error) { + return pidLabel(pid) +} + +// ExecLabel returns the SELinux label that the kernel will use for any programs +// that are executed by the current process thread, or an error. +func ExecLabel() (string, error) { + return execLabel() +} + +// CanonicalizeContext takes a context string and writes it to the kernel +// the function then returns the context that the kernel will use. Use this +// function to check if two contexts are equivalent +func CanonicalizeContext(val string) (string, error) { + return canonicalizeContext(val) +} + +// ComputeCreateContext requests the type transition from source to target for +// class from the kernel. +func ComputeCreateContext(source string, target string, class string) (string, error) { + return computeCreateContext(source, target, class) +} + +// CalculateGlbLub computes the glb (greatest lower bound) and lub (least upper bound) +// of a source and target range. +// The glblub is calculated as the greater of the low sensitivities and +// the lower of the high sensitivities and the and of each category bitset. +func CalculateGlbLub(sourceRange, targetRange string) (string, error) { + return calculateGlbLub(sourceRange, targetRange) +} + +// SetExecLabel sets the SELinux label that the kernel will use for any programs +// that are executed by the current process thread, or an error. +func SetExecLabel(label string) error { + return setExecLabel(label) +} + +// SetTaskLabel sets the SELinux label for the current thread, or an error. +// This requires the dyntransition permission. +func SetTaskLabel(label string) error { + return setTaskLabel(label) +} + +// SetSocketLabel takes a process label and tells the kernel to assign the +// label to the next socket that gets created +func SetSocketLabel(label string) error { + return setSocketLabel(label) +} + +// SocketLabel retrieves the current socket label setting +func SocketLabel() (string, error) { + return socketLabel() +} + +// PeerLabel retrieves the label of the client on the other side of a socket +func PeerLabel(fd uintptr) (string, error) { + return peerLabel(fd) +} + +// SetKeyLabel takes a process label and tells the kernel to assign the +// label to the next kernel keyring that gets created +func SetKeyLabel(label string) error { + return setKeyLabel(label) +} + +// KeyLabel retrieves the current kernel keyring label setting +func KeyLabel() (string, error) { + return keyLabel() +} + +// Get returns the Context as a string +func (c Context) Get() string { + return c.get() +} + +// NewContext creates a new Context struct from the specified label +func NewContext(label string) (Context, error) { + return newContext(label) +} + +// ClearLabels clears all reserved labels +func ClearLabels() { + clearLabels() +} + +// ReserveLabel reserves the MLS/MCS level component of the specified label +func ReserveLabel(label string) { + reserveLabel(label) +} + +// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled +func EnforceMode() int { + return enforceMode() +} + +// SetEnforceMode sets the current SELinux mode Enforcing, Permissive. +// Disabled is not valid, since this needs to be set at boot time. +func SetEnforceMode(mode int) error { + return setEnforceMode(mode) +} + +// DefaultEnforceMode returns the systems default SELinux mode Enforcing, +// Permissive or Disabled. Note this is is just the default at boot time. +// EnforceMode tells you the systems current mode. +func DefaultEnforceMode() int { + return defaultEnforceMode() +} + +// ReleaseLabel un-reserves the MLS/MCS Level field of the specified label, +// allowing it to be used by another process. +func ReleaseLabel(label string) { + releaseLabel(label) +} + +// ROFileLabel returns the specified SELinux readonly file label +func ROFileLabel() string { + return roFileLabel() +} + +// KVMContainerLabels returns the default processLabel and mountLabel to be used +// for kvm containers by the calling process. +func KVMContainerLabels() (string, string) { + return kvmContainerLabels() +} + +// InitContainerLabels returns the default processLabel and file labels to be +// used for containers running an init system like systemd by the calling process. +func InitContainerLabels() (string, string) { + return initContainerLabels() +} + +// ContainerLabels returns an allocated processLabel and fileLabel to be used for +// container labeling by the calling process. +func ContainerLabels() (processLabel string, fileLabel string) { + return containerLabels() +} + +// SecurityCheckContext validates that the SELinux label is understood by the kernel +func SecurityCheckContext(val string) error { + return securityCheckContext(val) +} + +// CopyLevel returns a label with the MLS/MCS level from src label replaced on +// the dest label. +func CopyLevel(src, dest string) (string, error) { + return copyLevel(src, dest) +} + +// Chcon changes the fpath file object to the SELinux label label. +// If fpath is a directory and recurse is true, then Chcon walks the +// directory tree setting the label. +func Chcon(fpath string, label string, recurse bool) error { + return chcon(fpath, label, recurse) +} + +// DupSecOpt takes an SELinux process label and returns security options that +// can be used to set the SELinux Type and Level for future container processes. +func DupSecOpt(src string) ([]string, error) { + return dupSecOpt(src) +} + +// DisableSecOpt returns a security opt that can be used to disable SELinux +// labeling support for future container processes. +func DisableSecOpt() []string { + return disableSecOpt() +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go index 9c979e5e2..d6b0d49db 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go @@ -20,17 +20,12 @@ import ( "github.com/opencontainers/selinux/pkg/pwalk" "github.com/pkg/errors" + "github.com/willf/bitset" "golang.org/x/sys/unix" ) const ( - // Enforcing constant indicate SELinux is in enforcing mode - Enforcing = 1 - // Permissive constant to indicate SELinux is in permissive mode - Permissive = 0 - // Disabled constant to indicate SELinux is disabled - Disabled = -1 - + minSensLen = 2 contextFile = "/usr/share/containers/selinux/contexts" selinuxDir = "/etc/selinux/" selinuxConfig = selinuxDir + "config" @@ -49,17 +44,27 @@ type selinuxState struct { sync.Mutex } -var ( - // ErrMCSAlreadyExists is returned when trying to allocate a duplicate MCS. - ErrMCSAlreadyExists = errors.New("MCS label already exists") - // ErrEmptyPath is returned when an empty path has been specified. - ErrEmptyPath = errors.New("empty path") - // InvalidLabel is returned when an invalid label is specified. - InvalidLabel = errors.New("Invalid Label") +type level struct { + sens uint + cats *bitset.BitSet +} - assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) - roFileLabel string - state = selinuxState{ +type mlsRange struct { + low *level + high *level +} + +type levelItem byte + +const ( + sensitivity levelItem = 's' + category levelItem = 'c' +) + +var ( + assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) + readOnlyFileLabel string + state = selinuxState{ mcsList: make(map[string]bool), } @@ -68,9 +73,6 @@ var ( haveThreadSelf bool ) -// Context is a representation of the SELinux label broken into 4 parts -type Context map[string]string - func (s *selinuxState) setEnable(enabled bool) bool { s.Lock() defer s.Unlock() @@ -97,8 +99,8 @@ func (s *selinuxState) getEnabled() bool { return s.setEnable(enabled) } -// SetDisabled disables selinux support for the package -func SetDisabled() { +// setDisabled disables SELinux support for the package +func setDisabled() { state.setEnable(false) } @@ -190,15 +192,15 @@ func (s *selinuxState) getSELinuxfs() string { // getSelinuxMountPoint returns the path to the mountpoint of an selinuxfs // filesystem or an empty string if no mountpoint is found. Selinuxfs is -// a proc-like pseudo-filesystem that exposes the selinux policy API to +// a proc-like pseudo-filesystem that exposes the SELinux policy API to // processes. The existence of an selinuxfs mount is used to determine -// whether selinux is currently enabled or not. +// whether SELinux is currently enabled or not. func getSelinuxMountPoint() string { return state.getSELinuxfs() } -// GetEnabled returns whether selinux is currently enabled. -func GetEnabled() bool { +// getEnabled returns whether SELinux is currently enabled. +func getEnabled() bool { return state.getEnabled() } @@ -282,8 +284,9 @@ func readCon(fpath string) (string, error) { return strings.Trim(retval, "\x00"), nil } -// ClassIndex returns the int index for an object class in the loaded policy, or -1 and an error -func ClassIndex(class string) (int, error) { +// classIndex returns the int index for an object class in the loaded policy, +// or -1 and an error +func classIndex(class string) (int, error) { permpath := fmt.Sprintf("class/%s/index", class) indexpath := filepath.Join(getSelinuxMountPoint(), permpath) @@ -299,8 +302,8 @@ func ClassIndex(class string) (int, error) { return index, nil } -// SetFileLabel sets the SELinux label for this path or returns an error. -func SetFileLabel(fpath string, label string) error { +// setFileLabel sets the SELinux label for this path or returns an error. +func setFileLabel(fpath string, label string) error { if fpath == "" { return ErrEmptyPath } @@ -310,8 +313,8 @@ func SetFileLabel(fpath string, label string) error { return nil } -// FileLabel returns the SELinux label for this path or returns an error. -func FileLabel(fpath string) (string, error) { +// fileLabel returns the SELinux label for this path or returns an error. +func fileLabel(fpath string) (string, error) { if fpath == "" { return "", ErrEmptyPath } @@ -327,37 +330,31 @@ func FileLabel(fpath string) (string, error) { return string(label), nil } -/* -SetFSCreateLabel tells kernel the label to create all file system objects -created by this task. Setting label="" to return to default. -*/ -func SetFSCreateLabel(label string) error { +// setFSCreateLabel tells kernel the label to create all file system objects +// created by this task. Setting label="" to return to default. +func setFSCreateLabel(label string) error { return writeAttr("fscreate", label) } -/* -FSCreateLabel returns the default label the kernel which the kernel is using -for file system objects created by this task. "" indicates default. -*/ -func FSCreateLabel() (string, error) { +// fsCreateLabel returns the default label the kernel which the kernel is using +// for file system objects created by this task. "" indicates default. +func fsCreateLabel() (string, error) { return readAttr("fscreate") } -// CurrentLabel returns the SELinux label of the current process thread, or an error. -func CurrentLabel() (string, error) { +// currentLabel returns the SELinux label of the current process thread, or an error. +func currentLabel() (string, error) { return readAttr("current") } -// PidLabel returns the SELinux label of the given pid, or an error. -func PidLabel(pid int) (string, error) { +// pidLabel returns the SELinux label of the given pid, or an error. +func pidLabel(pid int) (string, error) { return readCon(fmt.Sprintf("/proc/%d/attr/current", pid)) } -/* -ExecLabel returns the SELinux label that the kernel will use for any programs -that are executed by the current process thread, or an error. -*/ -func ExecLabel() (string, error) { +// ExecLabel returns the SELinux label that the kernel will use for any programs +// that are executed by the current process thread, or an error. +func execLabel() (string, error) { return readAttr("exec") } @@ -366,7 +363,7 @@ func writeCon(fpath, val string) error { return ErrEmptyPath } if val == "" { - if !GetEnabled() { + if !getEnabled() { return nil } } @@ -418,20 +415,17 @@ func writeAttr(attr, val string) error { return writeCon(attrPath(attr), val) } -/* -CanonicalizeContext takes a context string and writes it to the kernel -the function then returns the context that the kernel will use. This function -can be used to see if two contexts are equivalent -*/ -func CanonicalizeContext(val string) (string, error) { +// canonicalizeContext takes a context string and writes it to the kernel +// the function then returns the context that the kernel will use. Use this +// function to check if two contexts are equivalent +func canonicalizeContext(val string) (string, error) { return readWriteCon(filepath.Join(getSelinuxMountPoint(), "context"), val) } -/* -ComputeCreateContext requests the type transition from source to target for class from the kernel. -*/ -func ComputeCreateContext(source string, target string, class string) (string, error) { - classidx, err := ClassIndex(class) +// computeCreateContext requests the type transition from source to target for +// class from the kernel. +func computeCreateContext(source string, target string, class string) (string, error) { + classidx, err := classIndex(class) if err != nil { return "", err } @@ -439,6 +433,217 @@ func ComputeCreateContext(source string, target string, class string) (string, e return readWriteCon(filepath.Join(getSelinuxMountPoint(), "create"), fmt.Sprintf("%s %s %d", source, target, classidx)) } +// catsToBitset stores categories in a bitset. +func catsToBitset(cats string) (*bitset.BitSet, error) { + bitset := &bitset.BitSet{} + + catlist := strings.Split(cats, ",") + for _, r := range catlist { + ranges := strings.SplitN(r, ".", 2) + if len(ranges) > 1 { + catstart, err := parseLevelItem(ranges[0], category) + if err != nil { + return nil, err + } + catend, err := parseLevelItem(ranges[1], category) + if err != nil { + return nil, err + } + for i := catstart; i <= catend; i++ { + bitset.Set(i) + } + } else { + cat, err := parseLevelItem(ranges[0], category) + if err != nil { + return nil, err + } + bitset.Set(cat) + } + } + + return bitset, nil +} + +// parseLevelItem parses and verifies that a sensitivity or category are valid +func parseLevelItem(s string, sep levelItem) (uint, error) { + if len(s) < minSensLen || levelItem(s[0]) != sep { + return 0, ErrLevelSyntax + } + val, err := strconv.ParseUint(s[1:], 10, 32) + if err != nil { + return 0, err + } + + return uint(val), nil +} + +// parseLevel fills a level from a string that contains +// a sensitivity and categories +func (l *level) parseLevel(levelStr string) error { + lvl := strings.SplitN(levelStr, ":", 2) + sens, err := parseLevelItem(lvl[0], sensitivity) + if err != nil { + return errors.Wrap(err, "failed to parse sensitivity") + } + l.sens = sens + if len(lvl) > 1 { + cats, err := catsToBitset(lvl[1]) + if err != nil { + return errors.Wrap(err, "failed to parse categories") + } + l.cats = cats + } + + return nil +} + +// rangeStrToMLSRange marshals a string representation of a range. +func rangeStrToMLSRange(rangeStr string) (*mlsRange, error) { + mlsRange := &mlsRange{} + levelSlice := strings.SplitN(rangeStr, "-", 2) + + switch len(levelSlice) { + // rangeStr that has a low and a high level, e.g. s4:c0.c1023-s6:c0.c1023 + case 2: + mlsRange.high = &level{} + if err := mlsRange.high.parseLevel(levelSlice[1]); err != nil { + return nil, errors.Wrapf(err, "failed to parse high level %q", levelSlice[1]) + } + fallthrough + // rangeStr that is single level, e.g. s6:c0,c3,c5,c30.c1023 + case 1: + mlsRange.low = &level{} + if err := mlsRange.low.parseLevel(levelSlice[0]); err != nil { + return nil, errors.Wrapf(err, "failed to parse low level %q", levelSlice[0]) + } + } + + if mlsRange.high == nil { + mlsRange.high = mlsRange.low + } + + return mlsRange, nil +} + +// bitsetToStr takes a category bitset and returns it in the +// canonical selinux syntax +func bitsetToStr(c *bitset.BitSet) string { + var str string + i, e := c.NextSet(0) + len := 0 + for e { + if len == 0 { + if str != "" { + str += "," + } + str += "c" + strconv.Itoa(int(i)) + } + + next, e := c.NextSet(i + 1) + if e { + // consecutive cats + if next == i+1 { + len++ + i = next + continue + } + } + if len == 1 { + str += ",c" + strconv.Itoa(int(i)) + } else if len > 1 { + str += ".c" + strconv.Itoa(int(i)) + } + if !e { + break + } + len = 0 + i = next + } + + return str +} + +func (l1 *level) equal(l2 *level) bool { + if l2 == nil || l1 == nil { + return l1 == l2 + } + if l1.sens != l2.sens { + return false + } + return l1.cats.Equal(l2.cats) +} + +// String returns an mlsRange as a string. +func (m mlsRange) String() string { + low := "s" + strconv.Itoa(int(m.low.sens)) + if m.low.cats != nil && m.low.cats.Count() > 0 { + low += ":" + bitsetToStr(m.low.cats) + } + + if m.low.equal(m.high) { + return low + } + + high := "s" + strconv.Itoa(int(m.high.sens)) + if m.high.cats != nil && m.high.cats.Count() > 0 { + high += ":" + bitsetToStr(m.high.cats) + } + + return low + "-" + high +} + +func max(a, b uint) uint { + if a > b { + return a + } + return b +} + +func min(a, b uint) uint { + if a < b { + return a + } + return b +} + +// calculateGlbLub computes the glb (greatest lower bound) and lub (least upper bound) +// of a source and target range. +// The glblub is calculated as the greater of the low sensitivities and +// the lower of the high sensitivities and the and of each category bitset. +func calculateGlbLub(sourceRange, targetRange string) (string, error) { + s, err := rangeStrToMLSRange(sourceRange) + if err != nil { + return "", err + } + t, err := rangeStrToMLSRange(targetRange) + if err != nil { + return "", err + } + + if s.high.sens < t.low.sens || t.high.sens < s.low.sens { + /* these ranges have no common sensitivities */ + return "", ErrIncomparable + } + + outrange := &mlsRange{low: &level{}, high: &level{}} + + /* take the greatest of the low */ + outrange.low.sens = max(s.low.sens, t.low.sens) + + /* take the least of the high */ + outrange.high.sens = min(s.high.sens, t.high.sens) + + /* find the intersecting categories */ + if s.low.cats != nil && t.low.cats != nil { + outrange.low.cats = s.low.cats.Intersection(t.low.cats) + } + if s.high.cats != nil && t.high.cats != nil { + outrange.high.cats = s.high.cats.Intersection(t.high.cats) + } + + return outrange.String(), nil +} + func readWriteCon(fpath string, val string) (string, error) { if fpath == "" { return "", ErrEmptyPath @@ -461,41 +666,37 @@ func readWriteCon(fpath string, val string) (string, error) { return strings.Trim(retval, "\x00"), nil } -/* -SetExecLabel sets the SELinux label that the kernel will use for any programs -that are executed by the current process thread, or an error. -*/ -func SetExecLabel(label string) error { +// setExecLabel sets the SELinux label that the kernel will use for any programs +// that are executed by the current process thread, or an error. +func setExecLabel(label string) error { return writeAttr("exec", label) } -/* -SetTaskLabel sets the SELinux label for the current thread, or an error. -This requires the dyntransition permission. -*/ -func SetTaskLabel(label string) error { +// setTaskLabel sets the SELinux label for the current thread, or an error. +// This requires the dyntransition permission. +func setTaskLabel(label string) error { return writeAttr("current", label) } -// SetSocketLabel takes a process label and tells the kernel to assign the +// setSocketLabel takes a process label and tells the kernel to assign the // label to the next socket that gets created -func SetSocketLabel(label string) error { +func setSocketLabel(label string) error { return writeAttr("sockcreate", label) } -// SocketLabel retrieves the current socket label setting -func SocketLabel() (string, error) { +// socketLabel retrieves the current socket label setting +func socketLabel() (string, error) { return readAttr("sockcreate") } -// PeerLabel retrieves the label of the client on the other side of a socket -func PeerLabel(fd uintptr) (string, error) { +// peerLabel retrieves the label of the client on the other side of a socket +func peerLabel(fd uintptr) (string, error) { return unix.GetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_PEERSEC) } -// SetKeyLabel takes a process label and tells the kernel to assign the +// setKeyLabel takes a process label and tells the kernel to assign the // label to the next kernel keyring that gets created -func SetKeyLabel(label string) error { +func setKeyLabel(label string) error { err := writeCon("/proc/self/attr/keycreate", label) if os.IsNotExist(errors.Cause(err)) { return nil @@ -506,21 +707,21 @@ func SetKeyLabel(label string) error { return err } -// KeyLabel retrieves the current kernel keyring label setting -func KeyLabel() (string, error) { +// keyLabel retrieves the current kernel keyring label setting +func keyLabel() (string, error) { return readCon("/proc/self/attr/keycreate") } -// Get returns the Context as a string -func (c Context) Get() string { +// get returns the Context as a string +func (c Context) get() string { if c["level"] != "" { return fmt.Sprintf("%s:%s:%s:%s", c["user"], c["role"], c["type"], c["level"]) } return fmt.Sprintf("%s:%s:%s", c["user"], c["role"], c["type"]) } -// NewContext creates a new Context struct from the specified label -func NewContext(label string) (Context, error) { +// newContext creates a new Context struct from the specified label +func newContext(label string) (Context, error) { c := make(Context) if len(label) != 0 { @@ -538,15 +739,15 @@ func NewContext(label string) (Context, error) { return c, nil } -// ClearLabels clears all reserved labels -func ClearLabels() { +// clearLabels clears all reserved labels +func clearLabels() { state.Lock() state.mcsList = make(map[string]bool) state.Unlock() } -// ReserveLabel reserves the MLS/MCS level component of the specified label -func ReserveLabel(label string) { +// reserveLabel reserves the MLS/MCS level component of the specified label +func reserveLabel(label string) { if len(label) != 0 { con := strings.SplitN(label, ":", 4) if len(con) > 3 { @@ -559,8 +760,8 @@ func selinuxEnforcePath() string { return path.Join(getSelinuxMountPoint(), "enforce") } -// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled -func EnforceMode() int { +// enforceMode returns the current SELinux mode Enforcing, Permissive, Disabled +func enforceMode() int { var enforce int enforceB, err := ioutil.ReadFile(selinuxEnforcePath()) @@ -574,20 +775,16 @@ func EnforceMode() int { return enforce } -/* -SetEnforceMode sets the current SELinux mode Enforcing, Permissive. -Disabled is not valid, since this needs to be set at boot time. -*/ -func SetEnforceMode(mode int) error { +// setEnforceMode sets the current SELinux mode Enforcing, Permissive. +// Disabled is not valid, since this needs to be set at boot time. +func setEnforceMode(mode int) error { return ioutil.WriteFile(selinuxEnforcePath(), []byte(strconv.Itoa(mode)), 0644) } -/* -DefaultEnforceMode returns the systems default SELinux mode Enforcing, -Permissive or Disabled. Note this is is just the default at boot time. -EnforceMode tells you the systems current mode. -*/ -func DefaultEnforceMode() int { +// defaultEnforceMode returns the systems default SELinux mode Enforcing, +// Permissive or Disabled. Note this is is just the default at boot time. +// EnforceMode tells you the systems current mode. +func defaultEnforceMode() int { switch readConfig(selinuxTag) { case "enforcing": return Enforcing @@ -667,11 +864,9 @@ func uniqMcs(catRange uint32) string { return mcs } -/* -ReleaseLabel will unreserve the MLS/MCS Level field of the specified label. -Allowing it to be used by another process. -*/ -func ReleaseLabel(label string) { +// releaseLabel un-reserves the MLS/MCS Level field of the specified label, +// allowing it to be used by another process. +func releaseLabel(label string) { if len(label) != 0 { con := strings.SplitN(label, ":", 4) if len(con) > 3 { @@ -680,9 +875,9 @@ func ReleaseLabel(label string) { } } -// ROFileLabel returns the specified SELinux readonly file label -func ROFileLabel() string { - return roFileLabel +// roFileLabel returns the specified SELinux readonly file label +func roFileLabel() string { + return readOnlyFileLabel } func openContextFile() (*os.File, error) { @@ -737,11 +932,9 @@ func loadLabels() map[string]string { return labels } -/* -KVMContainerLabels returns the default processLabel and mountLabel to be used -for kvm containers by the calling process. -*/ -func KVMContainerLabels() (string, string) { +// kvmContainerLabels returns the default processLabel and mountLabel to be used +// for kvm containers by the calling process. +func kvmContainerLabels() (string, string) { processLabel := labels["kvm_process"] if processLabel == "" { processLabel = labels["process"] @@ -750,11 +943,9 @@ func KVMContainerLabels() (string, string) { return addMcs(processLabel, labels["file"]) } -/* -InitContainerLabels returns the default processLabel and file labels to be -used for containers running an init system like systemd by the calling process. -*/ -func InitContainerLabels() (string, string) { +// initContainerLabels returns the default processLabel and file labels to be +// used for containers running an init system like systemd by the calling process. +func initContainerLabels() (string, string) { processLabel := labels["init_process"] if processLabel == "" { processLabel = labels["process"] @@ -763,25 +954,23 @@ func InitContainerLabels() (string, string) { return addMcs(processLabel, labels["file"]) } -/* -ContainerLabels returns an allocated processLabel and fileLabel to be used for -container labeling by the calling process. -*/ -func ContainerLabels() (processLabel string, fileLabel string) { - if !GetEnabled() { +// containerLabels returns an allocated processLabel and fileLabel to be used for +// container labeling by the calling process. +func containerLabels() (processLabel string, fileLabel string) { + if !getEnabled() { return "", "" } processLabel = labels["process"] fileLabel = labels["file"] - roFileLabel = labels["ro_file"] + readOnlyFileLabel = labels["ro_file"] if processLabel == "" || fileLabel == "" { return "", fileLabel } - if roFileLabel == "" { - roFileLabel = fileLabel + if readOnlyFileLabel == "" { + readOnlyFileLabel = fileLabel } return addMcs(processLabel, fileLabel) @@ -790,7 +979,7 @@ func ContainerLabels() (processLabel string, fileLabel string) { func addMcs(processLabel, fileLabel string) (string, string) { scon, _ := NewContext(processLabel) if scon["level"] != "" { - mcs := uniqMcs(1024) + mcs := uniqMcs(CategoryRange) scon["level"] = mcs processLabel = scon.Get() scon, _ = NewContext(fileLabel) @@ -800,16 +989,14 @@ func addMcs(processLabel, fileLabel string) (string, string) { return processLabel, fileLabel } -// SecurityCheckContext validates that the SELinux label is understood by the kernel -func SecurityCheckContext(val string) error { +// securityCheckContext validates that the SELinux label is understood by the kernel +func securityCheckContext(val string) error { return ioutil.WriteFile(path.Join(getSelinuxMountPoint(), "context"), []byte(val), 0644) } -/* -CopyLevel returns a label with the MLS/MCS level from src label replaced on -the dest label. -*/ -func CopyLevel(src, dest string) (string, error) { +// copyLevel returns a label with the MLS/MCS level from src label replaced on +// the dest label. +func copyLevel(src, dest string) (string, error) { if src == "" { return "", nil } @@ -833,7 +1020,7 @@ func CopyLevel(src, dest string) (string, error) { return tcon.Get(), nil } -// Prevent users from relabing system files +// Prevent users from relabeling system files func badPrefix(fpath string) error { if fpath == "" { return ErrEmptyPath @@ -848,10 +1035,10 @@ func badPrefix(fpath string) error { return nil } -// Chcon changes the fpath file object to the SELinux label label. -// If fpath is a directory and recurse is true, Chcon will walk the +// chcon changes the fpath file object to the SELinux label label. +// If fpath is a directory and recurse is true, then chcon walks the // directory tree setting the label. -func Chcon(fpath string, label string, recurse bool) error { +func chcon(fpath string, label string, recurse bool) error { if fpath == "" { return ErrEmptyPath } @@ -876,9 +1063,9 @@ func Chcon(fpath string, label string, recurse bool) error { }) } -// DupSecOpt takes an SELinux process label and returns security options that +// dupSecOpt takes an SELinux process label and returns security options that // can be used to set the SELinux Type and Level for future container processes. -func DupSecOpt(src string) ([]string, error) { +func dupSecOpt(src string) ([]string, error) { if src == "" { return nil, nil } @@ -903,8 +1090,8 @@ func DupSecOpt(src string) ([]string, error) { return dup, nil } -// DisableSecOpt returns a security opt that can be used to disable SELinux +// disableSecOpt returns a security opt that can be used to disable SELinux // labeling support for future container processes. -func DisableSecOpt() []string { +func disableSecOpt() []string { return []string{"disable"} } diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go index f9f5e2061..c526b210f 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_stub.go @@ -2,253 +2,147 @@ package selinux -import ( - "errors" -) - -const ( - // Enforcing constant indicate SELinux is in enforcing mode - Enforcing = 1 - // Permissive constant to indicate SELinux is in permissive mode - Permissive = 0 - // Disabled constant to indicate SELinux is disabled - Disabled = -1 -) - -var ( - // ErrMCSAlreadyExists is returned when trying to allocate a duplicate MCS. - ErrMCSAlreadyExists = errors.New("MCS label already exists") - // ErrEmptyPath is returned when an empty path has been specified. - ErrEmptyPath = errors.New("empty path") -) - -// Context is a representation of the SELinux label broken into 4 parts -type Context map[string]string - -// SetDisabled disables selinux support for the package -func SetDisabled() { - return +func setDisabled() { } -// GetEnabled returns whether selinux is currently enabled. -func GetEnabled() bool { +func getEnabled() bool { return false } -// ClassIndex returns the int index for an object class in the loaded policy, or -1 and an error -func ClassIndex(class string) (int, error) { +func classIndex(class string) (int, error) { return -1, nil } -// SetFileLabel sets the SELinux label for this path or returns an error. -func SetFileLabel(fpath string, label string) error { +func setFileLabel(fpath string, label string) error { return nil } -// FileLabel returns the SELinux label for this path or returns an error. -func FileLabel(fpath string) (string, error) { +func fileLabel(fpath string) (string, error) { return "", nil } -/* -SetFSCreateLabel tells kernel the label to create all file system objects -created by this task. Setting label="" to return to default. -*/ -func SetFSCreateLabel(label string) error { +func setFSCreateLabel(label string) error { return nil } -/* -FSCreateLabel returns the default label the kernel which the kernel is using -for file system objects created by this task. "" indicates default. -*/ -func FSCreateLabel() (string, error) { +func fsCreateLabel() (string, error) { return "", nil } -// CurrentLabel returns the SELinux label of the current process thread, or an error. -func CurrentLabel() (string, error) { +func currentLabel() (string, error) { return "", nil } -// PidLabel returns the SELinux label of the given pid, or an error. -func PidLabel(pid int) (string, error) { +func pidLabel(pid int) (string, error) { return "", nil } -/* -ExecLabel returns the SELinux label that the kernel will use for any programs -that are executed by the current process thread, or an error. -*/ -func ExecLabel() (string, error) { +func execLabel() (string, error) { return "", nil } -/* -CanonicalizeContext takes a context string and writes it to the kernel -the function then returns the context that the kernel will use. This function -can be used to see if two contexts are equivalent -*/ -func CanonicalizeContext(val string) (string, error) { +func canonicalizeContext(val string) (string, error) { return "", nil } -/* -ComputeCreateContext requests the type transition from source to target for class from the kernel. -*/ -func ComputeCreateContext(source string, target string, class string) (string, error) { +func computeCreateContext(source string, target string, class string) (string, error) { return "", nil } -/* -SetExecLabel sets the SELinux label that the kernel will use for any programs -that are executed by the current process thread, or an error. -*/ -func SetExecLabel(label string) error { +func calculateGlbLub(sourceRange, targetRange string) (string, error) { + return "", nil +} + +func setExecLabel(label string) error { return nil } -/* -SetTaskLabel sets the SELinux label for the current thread, or an error. -This requires the dyntransition permission. -*/ -func SetTaskLabel(label string) error { +func setTaskLabel(label string) error { return nil } -/* -SetSocketLabel sets the SELinux label that the kernel will use for any programs -that are executed by the current process thread, or an error. -*/ -func SetSocketLabel(label string) error { +func setSocketLabel(label string) error { return nil } -// SocketLabel retrieves the current socket label setting -func SocketLabel() (string, error) { +func socketLabel() (string, error) { return "", nil } -// PeerLabel retrieves the label of the client on the other side of a socket -func PeerLabel(fd uintptr) (string, error) { +func peerLabel(fd uintptr) (string, error) { return "", nil } -// SetKeyLabel takes a process label and tells the kernel to assign the -// label to the next kernel keyring that gets created -func SetKeyLabel(label string) error { +func setKeyLabel(label string) error { return nil } -// KeyLabel retrieves the current kernel keyring label setting -func KeyLabel() (string, error) { +func keyLabel() (string, error) { return "", nil } -// Get returns the Context as a string -func (c Context) Get() string { +func (c Context) get() string { return "" } -// NewContext creates a new Context struct from the specified label -func NewContext(label string) (Context, error) { +func newContext(label string) (Context, error) { c := make(Context) return c, nil } -// ClearLabels clears all reserved MLS/MCS levels -func ClearLabels() { - return +func clearLabels() { } -// ReserveLabel reserves the MLS/MCS level component of the specified label -func ReserveLabel(label string) { - return +func reserveLabel(label string) { } -// EnforceMode returns the current SELinux mode Enforcing, Permissive, Disabled -func EnforceMode() int { +func enforceMode() int { return Disabled } -/* -SetEnforceMode sets the current SELinux mode Enforcing, Permissive. -Disabled is not valid, since this needs to be set at boot time. -*/ -func SetEnforceMode(mode int) error { +func setEnforceMode(mode int) error { return nil } -/* -DefaultEnforceMode returns the systems default SELinux mode Enforcing, -Permissive or Disabled. Note this is is just the default at boot time. -EnforceMode tells you the systems current mode. -*/ -func DefaultEnforceMode() int { +func defaultEnforceMode() int { return Disabled } -/* -ReleaseLabel will unreserve the MLS/MCS Level field of the specified label. -Allowing it to be used by another process. -*/ -func ReleaseLabel(label string) { - return +func releaseLabel(label string) { } -// ROFileLabel returns the specified SELinux readonly file label -func ROFileLabel() string { +func roFileLabel() string { return "" } -// KVMContainerLabels returns the default processLabel and mountLabel to be used -// for kvm containers by the calling process. -func KVMContainerLabels() (string, string) { +func kvmContainerLabels() (string, string) { return "", "" } -// InitContainerLabels returns the default processLabel and file labels to be -// used for containers running an init system like systemd by the calling -func InitContainerLabels() (string, string) { +func initContainerLabels() (string, string) { return "", "" } -/* -ContainerLabels returns an allocated processLabel and fileLabel to be used for -container labeling by the calling process. -*/ -func ContainerLabels() (processLabel string, fileLabel string) { +func containerLabels() (processLabel string, fileLabel string) { return "", "" } -// SecurityCheckContext validates that the SELinux label is understood by the kernel -func SecurityCheckContext(val string) error { +func securityCheckContext(val string) error { return nil } -/* -CopyLevel returns a label with the MLS/MCS level from src label replaced on -the dest label. -*/ -func CopyLevel(src, dest string) (string, error) { +func copyLevel(src, dest string) (string, error) { return "", nil } -// Chcon changes the `fpath` file object to the SELinux label `label`. -// If `fpath` is a directory and `recurse`` is true, Chcon will walk the -// directory tree setting the label. -func Chcon(fpath string, label string, recurse bool) error { +func chcon(fpath string, label string, recurse bool) error { return nil } -// DupSecOpt takes an SELinux process label and returns security options that -// can be used to set the SELinux Type and Level for future container processes. -func DupSecOpt(src string) ([]string, error) { +func dupSecOpt(src string) ([]string, error) { return nil, nil } -// DisableSecOpt returns a security opt that can be used to disable SELinux -// labeling support for future container processes. -func DisableSecOpt() []string { +func disableSecOpt() []string { return []string{"disable"} } diff --git a/vendor/github.com/opencontainers/selinux/go.mod b/vendor/github.com/opencontainers/selinux/go.mod index eb4053ee5..7bda0670f 100644 --- a/vendor/github.com/opencontainers/selinux/go.mod +++ b/vendor/github.com/opencontainers/selinux/go.mod @@ -3,6 +3,7 @@ module github.com/opencontainers/selinux go 1.13 require ( - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 + github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 golang.org/x/sys v0.0.0-20191115151921-52ab43148777 ) diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go index 2ee0d0150..63fde1842 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go @@ -48,7 +48,11 @@ func WalkN(root string, walkFn WalkFunc, num int) error { errCh := make(chan error, 1) // get the first error, ignore others // Start walking a tree asap - var err error + var ( + err error + wg sync.WaitGroup + ) + wg.Add(1) go func() { err = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { if err != nil { @@ -68,9 +72,9 @@ func WalkN(root string, walkFn WalkFunc, num int) error { if err == nil { close(files) } + wg.Done() }() - var wg sync.WaitGroup wg.Add(num) for i := 0; i < num; i++ { go func() { diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/willf/bitset/LICENSE new file mode 100644 index 000000000..59cab8a93 --- /dev/null +++ b/vendor/github.com/willf/bitset/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bitset/README.md b/vendor/github.com/willf/bitset/README.md new file mode 100644 index 000000000..6c62b20c6 --- /dev/null +++ b/vendor/github.com/willf/bitset/README.md @@ -0,0 +1,96 @@ +# bitset + +*Go language library to map between non-negative integers and boolean values* + +[![Master Build Status](https://secure.travis-ci.org/willf/bitset.png?branch=master)](https://travis-ci.org/willf/bitset?branch=master) +[![Master Coverage Status](https://coveralls.io/repos/willf/bitset/badge.svg?branch=master&service=github)](https://coveralls.io/github/willf/bitset?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) +[![GoDoc](https://godoc.org/github.com/willf/bitset?status.svg)](http://godoc.org/github.com/willf/bitset) + + +## Description + +Package bitset implements bitsets, a mapping between non-negative integers and boolean values. +It should be more efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing individual integers. + +But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. + +Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. + +### Example use: + +```go +package main + +import ( + "fmt" + "math/rand" + + "github.com/willf/bitset" +) + +func main() { + fmt.Printf("Hello from BitSet!\n") + var b bitset.BitSet + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` + +As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. + +Godoc documentation is at: https://godoc.org/github.com/willf/bitset + + +## Implementation Note + +Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. + +It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. + +## Installation + +```bash +go get github.com/willf/bitset +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +This project include a Makefile that allows you to test and build the project with simple commands. +To see all available options: +```bash +make help +``` + +## Running all tests + +Before committing the code, please check if it passes all tests using (note: this will install some dependencies): +```bash +make qa +``` diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go new file mode 100644 index 000000000..22e5d42e5 --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset.go @@ -0,0 +1,879 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. + +*/ +package bitset + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" +) + +// the wordSize of a bit set +const wordSize = uint(64) + +// log2WordSize is lg(wordSize) +const log2WordSize = uint(6) + +// allBits has every bit set +const allBits uint64 = 0xffffffffffffffff + +// default binary BigEndian +var binaryOrder binary.ByteOrder = binary.BigEndian + +// default json encoding base64.URLEncoding +var base64Encoding = base64.URLEncoding + +// Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) +func Base64StdEncoding() { base64Encoding = base64.StdEncoding } + +// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian) +func LittleEndian() { binaryOrder = binary.LittleEndian } + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet struct { + length uint + set []uint64 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet) safeSet() []uint64 { + if b.set == nil { + b.set = make([]uint64, wordsNeeded(0)) + } + return b.set +} + +// From is a constructor used to create a BitSet from an array of integers +func From(buf []uint64) *BitSet { + return &BitSet{uint(len(buf)) * 64, buf} +} + +// Bytes returns the bitset as array of integers +func (b *BitSet) Bytes() []uint64 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > (Cap() - wordSize + 1) { + return int(Cap() >> log2WordSize) + } + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// New creates a new BitSet with a hint that length bits will be required +func New(length uint) (bset *BitSet) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet{ + 0, + make([]uint64, 0), + } + } + }() + + bset = &BitSet{ + length, + make([]uint64, wordsNeeded(length)), + } + + return bset +} + +// Cap returns the total possible capacity, or number of bits +func Cap() uint { + return ^uint(0) +} + +// Len returns the number of bits in the BitSet. +// Note the difference to method Count, see example. +func (b *BitSet) Len() uint { + return b.length +} + +// extendSetMaybe adds additional words to incorporate new bits if needed +func (b *BitSet) extendSetMaybe(i uint) { + if i >= b.length { // if we need more bits, make 'em + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint64, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 + } +} + +// Test whether bit i is set. +func (b *BitSet) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 +} + +// Set bit i to 1 +func (b *BitSet) Set(i uint) *BitSet { + b.extendSetMaybe(i) + b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) + return b +} + +// Clear bit i to 0 +func (b *BitSet) Clear(i uint) *BitSet { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) + return b +} + +// SetTo sets bit i to value +func (b *BitSet) SetTo(i uint, value bool) *BitSet { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i +func (b *BitSet) Flip(i uint) *BitSet { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) + return b +} + +// Shrink shrinks BitSet to desired length in bits. It clears all bits > length +// and reduces the size and length of the set. +// +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +func (b *BitSet) Shrink(length uint) *BitSet { + idx := wordsNeeded(length + 1) + if idx > len(b.set) { + return b + } + shrunk := make([]uint64, idx) + copy(shrunk, b.set[:idx]) + b.set = shrunk + b.length = length + 1 + b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)) - 1)) + return b +} + +// InsertAt takes an index which indicates where a bit should be +// inserted. Then it shifts all the bits in the set to the left by 1, starting +// from the given index position, and sets the index position to 0. +// +// Depending on the size of your BitSet, and where you are inserting the new entry, +// this method could be extremely slow and in some cases might cause the entire BitSet +// to be recopied. +func (b *BitSet) InsertAt(idx uint) *BitSet { + insertAtElement := (idx >> log2WordSize) + + // if length of set is a multiple of wordSize we need to allocate more space first + if b.isLenExactMultiple() { + b.set = append(b.set, uint64(0)) + } + + var i uint + for i = uint(len(b.set) - 1); i > insertAtElement; i-- { + // all elements above the position where we want to insert can simply by shifted + b.set[i] <<= 1 + + // we take the most significant bit of the previous element and set it as + // the least significant bit of the current element + b.set[i] |= (b.set[i-1] & 0x8000000000000000) >> 63 + } + + // generate a mask to extract the data that we need to shift left + // within the element where we insert a bit + dataMask := ^(uint64(1)< 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// DeleteAt deletes the bit at the given index position from +// within the bitset +// All the bits residing on the left of the deleted bit get +// shifted right by 1 +// The running time of this operation may potentially be +// relatively slow, O(length) +func (b *BitSet) DeleteAt(i uint) *BitSet { + // the index of the slice element where we'll delete a bit + deleteAtElement := i >> log2WordSize + + // generate a mask for the data that needs to be shifted right + // within that slice element that gets modified + dataMask := ^((uint64(1) << (i & (wordSize - 1))) - 1) + + // extract the data that we'll shift right from the slice element + data := b.set[deleteAtElement] & dataMask + + // set the masked area to 0 while leaving the rest as it is + b.set[deleteAtElement] &= ^dataMask + + // shift the previously extracted data to the right and then + // set it in the previously masked area + b.set[deleteAtElement] |= (data >> 1) & dataMask + + // loop over all the consecutive slice elements to copy each + // lowest bit into the highest position of the previous element, + // then shift the entire content to the right by 1 + for i := int(deleteAtElement) + 1; i < len(b.set); i++ { + b.set[i-1] |= (b.set[i] & 1) << 63 + b.set[i] >>= 1 + } + + b.length = b.length - 1 + + return b +} + +// NextSet returns the next bit set from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no set bit found) +// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +func (b *BitSet) NextSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + if w != 0 { + return i + trailingZeroes64(w), true + } + x = x + 1 + for x < len(b.set) { + if b.set[x] != 0 { + return uint(x)*wordSize + trailingZeroes64(b.set[x]), true + } + x = x + 1 + + } + return 0, false +} + +// NextSetMany returns many next bit sets from the specified index, +// including possibly the current index and up to cap(buffer). +// If the returned slice has len zero, then no more set bits were found +// +// buffer := make([]uint, 256) // this should be reused +// j := uint(0) +// j, buffer = bitmap.NextSetMany(j, buffer) +// for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j,buffer) { +// for k := range buffer { +// do something with buffer[k] +// } +// j += 1 +// } +// +func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { + myanswer := buffer + capacity := cap(buffer) + x := int(i >> log2WordSize) + if x >= len(b.set) || capacity == 0 { + return 0, myanswer[:0] + } + skip := i & (wordSize - 1) + word := b.set[x] >> skip + myanswer = myanswer[:capacity] + size := int(0) + for word != 0 { + r := trailingZeroes64(word) + t := word & ((^word) + 1) + myanswer[size] = r + i + size++ + if size == capacity { + goto End + } + word = word ^ t + } + x++ + for idx, word := range b.set[x:] { + for word != 0 { + r := trailingZeroes64(word) + t := word & ((^word) + 1) + myanswer[size] = r + (uint(x+idx) << 6) + size++ + if size == capacity { + goto End + } + word = word ^ t + } + } +End: + if size > 0 { + return myanswer[size-1], myanswer[:size] + } + return 0, myanswer[:0] +} + +// NextClear returns the next clear bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are set) +func (b *BitSet) NextClear(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + wA := allBits >> (i & (wordSize - 1)) + index := i + trailingZeroes64(^w) + if w != wA && index < b.length { + return index, true + } + x++ + for x < len(b.set) { + index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) + if b.set[x] != allBits && index < b.length { + return index, true + } + x++ + } + return 0, false +} + +// ClearAll clears the entire BitSet +func (b *BitSet) ClearAll() *BitSet { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = 0 + } + } + return b +} + +// wordCount returns the number of words used in a bit set +func (b *BitSet) wordCount() int { + return len(b.set) +} + +// Clone this BitSet +func (b *BitSet) Clone() *BitSet { + c := New(b.length) + if b.set != nil { // Clone should not modify current object + copy(c.set, b.set) + } + return c +} + +// Copy into a destination BitSet +// Returning the size of the destination BitSet +// like array copy +func (b *BitSet) Copy(c *BitSet) (count uint) { + if c == nil { + return + } + if b.set != nil { // Copy should not modify current object + copy(c.set, b.set) + } + count = c.length + if b.length < c.length { + count = b.length + } + return +} + +// Count (number of set bits). +// Also known as "popcount" or "popularity count". +func (b *BitSet) Count() uint { + if b != nil && b.set != nil { + return uint(popcntSlice(b.set)) + } + return 0 +} + +// Equal tests the equivalence of two BitSets. +// False if they are of different sizes, otherwise true +// only if all the same bits are set +func (b *BitSet) Equal(c *BitSet) bool { + if c == nil || b == nil { + return c == b + } + if b.length != c.length { + return false + } + if b.length == 0 { // if they have both length == 0, then could have nil set + return true + } + // testing for equality shoud not transform the bitset (no call to safeSet) + + for p, v := range b.set { + if c.set[p] != v { + return false + } + } + return true +} + +func panicIfNull(b *BitSet) { + if b == nil { + panic(Error("BitSet must not be null")) + } +} + +// Difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + result = b.Clone() // clone b (in case b is bigger than compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the differnce +func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + cnt := uint64(0) + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) InPlaceDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &^= compare.set[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the union +func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet) InPlaceIntersection(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &= compare.set[i] + } + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet) Union(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet) UnionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntOrSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet) InPlaceUnion(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] |= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntXorSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] ^= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet) isLenExactMultiple() bool { + return b.length%wordSize == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet) cleanLastWord() { + if !b.isLenExactMultiple() { + b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) + } +} + +// Complement computes the (local) complement of a biset (up to length bits) +func (b *BitSet) Complement() (result *BitSet) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + return true + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet) IsSuperSet(other *BitSet) bool { + for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { + if !b.Test(i) { + return false + } + } + return true +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits +func (b *BitSet) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return buffer.String() +} + +// BinaryStorageSize returns the binary storage requirements +func (b *BitSet) BinaryStorageSize() int { + return binary.Size(uint64(0)) + binary.Size(b.set) +} + +// WriteTo writes a BitSet to a stream +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + + // Write length + err := binary.Write(stream, binaryOrder, length) + if err != nil { + return 0, err + } + + // Write set + err = binary.Write(stream, binaryOrder, b.set) + return int64(b.BinaryStorageSize()), err +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + + // Read length first + err := binary.Read(stream, binaryOrder, &length) + if err != nil { + return 0, err + } + newset := New(uint(length)) + + if uint64(newset.length) != length { + return 0, errors.New("Unmarshalling error: type mismatch") + } + + // Read remaining bytes as set + err = binary.Read(stream, binaryOrder, newset.set) + if err != nil { + return 0, err + } + + *b = *newset + return int64(b.BinaryStorageSize()), nil +} + +// MarshalBinary encodes a BitSet into a binary form and returns the result. +func (b *BitSet) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + + _, err := b.WriteTo(writer) + if err != nil { + return []byte{}, err + } + + err = writer.Flush() + + return buf.Bytes(), err +} + +// UnmarshalBinary decodes the binary form generated by MarshalBinary. +func (b *BitSet) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + reader := bufio.NewReader(buf) + + _, err := b.ReadFrom(reader) + + return err +} + +// MarshalJSON marshals a BitSet as a JSON structure +func (b *BitSet) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) + _, err := b.WriteTo(buffer) + if err != nil { + return nil, err + } + + // URLEncode all bytes + return json.Marshal(base64Encoding.EncodeToString(buffer.Bytes())) +} + +// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON +func (b *BitSet) UnmarshalJSON(data []byte) error { + // Unmarshal as string + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + // URLDecode string + buf, err := base64Encoding.DecodeString(s) + if err != nil { + return err + } + + _, err = b.ReadFrom(bytes.NewReader(buf)) + return err +} diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/willf/bitset/popcnt.go new file mode 100644 index 000000000..76577a838 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt.go @@ -0,0 +1,53 @@ +package bitset + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) (n uint64) { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/willf/bitset/popcnt_19.go b/vendor/github.com/willf/bitset/popcnt_19.go new file mode 100644 index 000000000..fc8ff4f36 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_19.go @@ -0,0 +1,45 @@ +// +build go1.9 + +package bitset + +import "math/bits" + +func popcntSlice(s []uint64) uint64 { + var cnt int + for _, x := range s { + cnt += bits.OnesCount64(x) + } + return uint64(cnt) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] &^ m[i]) + } + return uint64(cnt) +} + +func popcntAndSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] & m[i]) + } + return uint64(cnt) +} + +func popcntOrSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] | m[i]) + } + return uint64(cnt) +} + +func popcntXorSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] ^ m[i]) + } + return uint64(cnt) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/willf/bitset/popcnt_amd64.go new file mode 100644 index 000000000..4cf64f24a --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.go @@ -0,0 +1,68 @@ +// +build !go1.9 +// +build amd64,!appengine + +package bitset + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/willf/bitset/popcnt_amd64.s new file mode 100644 index 000000000..666c0dcc1 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.s @@ -0,0 +1,104 @@ +// +build !go1.9 +// +build amd64,!appengine + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/willf/bitset/popcnt_generic.go new file mode 100644 index 000000000..21e0ff7b4 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_generic.go @@ -0,0 +1,24 @@ +// +build !go1.9 +// +build !amd64 appengine + +package bitset + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_18.go b/vendor/github.com/willf/bitset/trailing_zeros_18.go new file mode 100644 index 000000000..c52b61be9 --- /dev/null +++ b/vendor/github.com/willf/bitset/trailing_zeros_18.go @@ -0,0 +1,14 @@ +// +build !go1.9 + +package bitset + +var deBruijn = [...]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +func trailingZeroes64(v uint64) uint { + return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) +} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_19.go b/vendor/github.com/willf/bitset/trailing_zeros_19.go new file mode 100644 index 000000000..36a988e71 --- /dev/null +++ b/vendor/github.com/willf/bitset/trailing_zeros_19.go @@ -0,0 +1,9 @@ +// +build go1.9 + +package bitset + +import "math/bits" + +func trailingZeroes64(v uint64) uint { + return uint(bits.TrailingZeros64(v)) +}