diff --git a/vendor.conf b/vendor.conf index 9eec16e6c..917e5a662 100644 --- a/vendor.conf +++ b/vendor.conf @@ -8,7 +8,7 @@ github.com/tchap/go-patricia 666120de432aea38ab06bd5c818f github.com/beorn7/perks 37c8de3658fcb183f997c4e13e8337516ab753e6 # v1.0.1 github.com/BurntSushi/toml 3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005 # v0.3.1 github.com/cespare/xxhash/v2 d7df74196a9e781ede915320c11c378c1b2f3a1f # v2.1.1 -github.com/containerd/cgroups 7347743e5d1e8500d9f27c8e748e689ed991d92b +github.com/containerd/cgroups b4448137398923af7f4918b8b2ad8249172ca7a6 github.com/containerd/console 8375c3424e4d7b114e8a90a4a40c8e1b40d1d4e6 # v1.0.0 github.com/containerd/containerd 01310155947cb6eec37dcae29742a165e56acb4a github.com/containerd/continuity 0ec596719c75bfd42908850990acea594b7593ac diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go index c198c5288..3f7dc91cb 100644 --- a/vendor/github.com/containerd/cgroups/blkio.go +++ b/vendor/github.com/containerd/cgroups/blkio.go @@ -90,54 +90,47 @@ func (b *blkioController) Update(path string, resources *specs.LinuxResources) e func (b *blkioController) Stat(path string, stats *v1.Metrics) error { stats.Blkio = &v1.BlkIOStat{} - settings := []blkioStatSettings{ - { - name: "throttle.io_serviced", - entry: &stats.Blkio.IoServicedRecursive, - }, - { - name: "throttle.io_service_bytes", - entry: &stats.Blkio.IoServiceBytesRecursive, - }, - } + + var settings []blkioStatSettings + // Try to read CFQ stats available on all CFQ enabled kernels first if _, err := os.Lstat(filepath.Join(b.Path(path), fmt.Sprintf("blkio.io_serviced_recursive"))); err == nil { - settings = []blkioStatSettings{} - settings = append(settings, - blkioStatSettings{ + settings = []blkioStatSettings{ + { name: "sectors_recursive", entry: &stats.Blkio.SectorsRecursive, }, - blkioStatSettings{ + { name: "io_service_bytes_recursive", entry: &stats.Blkio.IoServiceBytesRecursive, }, - blkioStatSettings{ + { name: "io_serviced_recursive", entry: &stats.Blkio.IoServicedRecursive, }, - blkioStatSettings{ + { name: "io_queued_recursive", entry: &stats.Blkio.IoQueuedRecursive, }, - blkioStatSettings{ + { name: "io_service_time_recursive", entry: &stats.Blkio.IoServiceTimeRecursive, }, - blkioStatSettings{ + { name: "io_wait_time_recursive", entry: &stats.Blkio.IoWaitTimeRecursive, }, - blkioStatSettings{ + { name: "io_merged_recursive", entry: &stats.Blkio.IoMergedRecursive, }, - blkioStatSettings{ + { name: "time_recursive", entry: &stats.Blkio.IoTimeRecursive, }, - ) + } } + f, err := os.Open(filepath.Join(b.procRoot, "diskstats")) if err != nil { return err @@ -149,6 +142,29 @@ func (b *blkioController) Stat(path string, stats *v1.Metrics) error { return err } + var size int + for _, t := range settings { + if err := b.readEntry(devices, path, t.name, t.entry); err != nil { + return err + } + size += len(*t.entry) + } + if size > 0 { + return nil + } + + // Even the kernel is compiled with the CFQ scheduler, the cgroup may not use + // block devices with the CFQ scheduler. If so, we should fallback to throttle.* files. + settings = []blkioStatSettings{ + { + name: "throttle.io_serviced", + entry: &stats.Blkio.IoServicedRecursive, + }, + { + name: "throttle.io_service_bytes", + entry: &stats.Blkio.IoServiceBytesRecursive, + }, + } for _, t := range settings { if err := b.readEntry(devices, path, t.name, t.entry); err != nil { return err @@ -165,9 +181,6 @@ func (b *blkioController) readEntry(devices map[deviceKey]string, path, name str defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { - if err := sc.Err(); err != nil { - return err - } // format: dev type amount fields := strings.FieldsFunc(sc.Text(), splitBlkIOStatLine) if len(fields) < 3 { @@ -204,7 +217,7 @@ func (b *blkioController) readEntry(devices map[deviceKey]string, path, name str Value: v, }) } - return nil + return sc.Err() } func createBlkioSettings(blkio *specs.LinuxBlockIO) []blkioSettings { diff --git a/vendor/github.com/containerd/cgroups/cpu.go b/vendor/github.com/containerd/cgroups/cpu.go index a22389f5c..ba8dda834 100644 --- a/vendor/github.com/containerd/cgroups/cpu.go +++ b/vendor/github.com/containerd/cgroups/cpu.go @@ -110,9 +110,6 @@ func (c *cpuController) Stat(path string, stats *v1.Metrics) error { // get or create the cpu field because cpuacct can also set values on this struct sc := bufio.NewScanner(f) for sc.Scan() { - if err := sc.Err(); err != nil { - return err - } key, v, err := parseKV(sc.Text()) if err != nil { return err @@ -126,5 +123,5 @@ func (c *cpuController) Stat(path string, stats *v1.Metrics) error { stats.CPU.Throttling.ThrottledTime = v } } - return nil + return sc.Err() } diff --git a/vendor/github.com/containerd/cgroups/go.mod b/vendor/github.com/containerd/cgroups/go.mod index 8b54fc1b0..e6257f69e 100644 --- a/vendor/github.com/containerd/cgroups/go.mod +++ b/vendor/github.com/containerd/cgroups/go.mod @@ -1,16 +1,19 @@ module github.com/containerd/cgroups -go 1.12 +go 1.13 require ( - github.com/cilium/ebpf v0.0.0-20191113100448-d9fb101ca1fb - github.com/coreos/go-systemd/v22 v22.0.0-20191111152658-2d78030078ef + github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3 + github.com/coreos/go-systemd/v22 v22.0.0 + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/docker/go-units v0.4.0 github.com/godbus/dbus/v5 v5.0.3 - github.com/gogo/protobuf v1.2.1 + github.com/gogo/protobuf v1.3.1 + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 - github.com/urfave/cli v1.22.1 - golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea + github.com/stretchr/testify v1.2.2 + github.com/urfave/cli v1.22.2 + golang.org/x/sys v0.0.0-20200120151820-655fe14d7479 ) diff --git a/vendor/github.com/containerd/cgroups/memory.go b/vendor/github.com/containerd/cgroups/memory.go index 9d55b4e52..f39c008b2 100644 --- a/vendor/github.com/containerd/cgroups/memory.go +++ b/vendor/github.com/containerd/cgroups/memory.go @@ -236,9 +236,6 @@ func (m *memoryController) parseStats(r io.Reader, stat *v1.MemoryStat) error { line int ) for sc.Scan() { - if err := sc.Err(); err != nil { - return err - } key, v, err := parseKV(sc.Text()) if err != nil { return fmt.Errorf("%d: %v", line, err) @@ -246,6 +243,9 @@ func (m *memoryController) parseStats(r io.Reader, stat *v1.MemoryStat) error { raw[key] = v line++ } + if err := sc.Err(); err != nil { + return err + } stat.Cache = raw["cache"] stat.RSS = raw["rss"] stat.RSSHuge = raw["rss_huge"] diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go index 6b12c928e..1e169d67a 100644 --- a/vendor/github.com/containerd/cgroups/utils.go +++ b/vendor/github.com/containerd/cgroups/utils.go @@ -181,6 +181,10 @@ func readPids(path string, subsystem Name) ([]Process, error) { }) } } + if err := s.Err(); err != nil { + // failed to read all pids? + return nil, err + } return out, nil } @@ -208,6 +212,9 @@ func readTasksPids(path string, subsystem Name) ([]Task, error) { }) } } + if err := s.Err(); err != nil { + return nil, err + } return out, nil } @@ -286,9 +293,6 @@ func parseCgroupFromReader(r io.Reader) (map[string]string, error) { s = bufio.NewScanner(r) ) for s.Scan() { - if err := s.Err(); err != nil { - return nil, err - } var ( text = s.Text() parts = strings.SplitN(text, ":", 3) @@ -302,6 +306,9 @@ func parseCgroupFromReader(r io.Reader) (map[string]string, error) { } } } + if err := s.Err(); err != nil { + return nil, err + } return cgroups, nil } @@ -313,16 +320,23 @@ func getCgroupDestination(subsystem string) (string, error) { defer f.Close() s := bufio.NewScanner(f) for s.Scan() { - if err := s.Err(); err != nil { - return "", err + fields := strings.Split(s.Text(), " ") + if len(fields) < 10 { + // broken mountinfo? + continue + } + if fields[len(fields)-3] != "cgroup" { + continue } - fields := strings.Fields(s.Text()) for _, opt := range strings.Split(fields[len(fields)-1], ",") { if opt == subsystem { return fields[3], nil } } } + if err := s.Err(); err != nil { + return "", err + } return "", ErrNoCgroupMountDestination } diff --git a/vendor/github.com/containerd/cgroups/v1.go b/vendor/github.com/containerd/cgroups/v1.go index a076d4692..2ec215c06 100644 --- a/vendor/github.com/containerd/cgroups/v1.go +++ b/vendor/github.com/containerd/cgroups/v1.go @@ -54,28 +54,20 @@ func v1MountPoint() (string, error) { defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { - if err := scanner.Err(); err != nil { - return "", err - } var ( - text = scanner.Text() - fields = strings.Split(text, " ") - // safe as mountinfo encodes mountpoints with spaces as \040. - index = strings.Index(text, " - ") - postSeparatorFields = strings.Fields(text[index+3:]) - numPostFields = len(postSeparatorFields) + text = scanner.Text() + fields = strings.Split(text, " ") + numFields = len(fields) ) - // this is an error as we can't detect if the mount is for "cgroup" - if numPostFields == 0 { - return "", fmt.Errorf("Found no fields post '-' in %q", text) + if numFields < 10 { + return "", fmt.Errorf("mountinfo: bad entry %q", text) } - if postSeparatorFields[0] == "cgroup" { - // check that the mount is properly formated. - if numPostFields < 3 { - return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text) - } + if fields[numFields-3] == "cgroup" { return filepath.Dir(fields[4]), nil } } + if err := scanner.Err(); err != nil { + return "", err + } return "", ErrMountPointNotExist } diff --git a/vendor/github.com/containerd/cgroups/v2/hugetlb.go b/vendor/github.com/containerd/cgroups/v2/hugetlb.go new file mode 100644 index 000000000..16b35bd78 --- /dev/null +++ b/vendor/github.com/containerd/cgroups/v2/hugetlb.go @@ -0,0 +1,37 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v2 + +import "strings" + +type HugeTlb []HugeTlbEntry + +type HugeTlbEntry struct { + HugePageSize string + Limit uint64 +} + +func (r *HugeTlb) Values() (o []Value) { + for _, e := range *r { + o = append(o, Value{ + filename: strings.Join([]string{"hugetlb", e.HugePageSize, "max"}, "."), + value: e.Limit, + }) + } + + return o +} diff --git a/vendor/github.com/containerd/cgroups/v2/manager.go b/vendor/github.com/containerd/cgroups/v2/manager.go index bd947a562..7b984dd5a 100644 --- a/vendor/github.com/containerd/cgroups/v2/manager.go +++ b/vendor/github.com/containerd/cgroups/v2/manager.go @@ -66,11 +66,12 @@ type Event struct { // Resources for a cgroups v2 unified hierarchy type Resources struct { - CPU *CPU - Memory *Memory - Pids *Pids - IO *IO - RDMA *RDMA + CPU *CPU + Memory *Memory + Pids *Pids + IO *IO + RDMA *RDMA + HugeTlb *HugeTlb // When len(Devices) is zero, devices are not controlled Devices []specs.LinuxDeviceCgroup } @@ -93,6 +94,9 @@ func (r *Resources) Values() (o []Value) { if r.RDMA != nil { o = append(o, r.RDMA.Values()...) } + if r.HugeTlb != nil { + o = append(o, r.HugeTlb.Values()...) + } return o } @@ -114,6 +118,9 @@ func (r *Resources) EnabledControllers() (c []string) { if r.RDMA != nil { c = append(c, "rdma") } + if r.HugeTlb != nil { + c = append(c, "hugetlb") + } return } @@ -422,11 +429,11 @@ func (c *Manager) Stat() (*stats.Metrics, error) { } metrics.Io = &stats.IOStat{Usage: readIoStats(c.path)} - metrics.Rdma = &stats.RdmaStat{ Current: rdmaStats(filepath.Join(c.path, "rdma.current")), Limit: rdmaStats(filepath.Join(c.path, "rdma.max")), } + metrics.Hugetlb = readHugeTlbStats(c.path) return &metrics, nil } @@ -489,16 +496,13 @@ func readKVStatsFile(path string, file string, out map[string]interface{}) error s := bufio.NewScanner(f) for s.Scan() { - if err := s.Err(); err != nil { - return err - } name, value, err := parseKV(s.Text()) if err != nil { return errors.Wrapf(err, "error while parsing %s (line=%q)", filepath.Join(path, file), s.Text()) } out[name] = value } - return nil + return s.Err() } func (c *Manager) Freeze() error { @@ -526,20 +530,20 @@ func (c *Manager) freeze(path string, state State) error { } } -func (c *Manager) MemoryEventFD() (uintptr, error) { +// MemoryEventFD returns inotify file descriptor and 'memory.events' inotify watch descriptor +func (c *Manager) MemoryEventFD() (int, uint32, error) { fpath := filepath.Join(c.path, "memory.events") fd, err := syscall.InotifyInit() if err != nil { - return 0, errors.Errorf("Failed to create inotify fd") + return 0, 0, errors.Errorf("Failed to create inotify fd") } - defer syscall.Close(fd) wd, err := syscall.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) if wd < 0 { - return 0, errors.Errorf("Failed to add inotify watch for %q", fpath) + syscall.Close(fd) + return 0, 0, errors.Errorf("Failed to add inotify watch for %q", fpath) } - defer syscall.InotifyRmWatch(fd, uint32(wd)) - return uintptr(fd), nil + return fd, uint32(wd), nil } func (c *Manager) EventChan() (<-chan Event, <-chan error) { @@ -551,15 +555,22 @@ func (c *Manager) EventChan() (<-chan Event, <-chan error) { } func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { - fd, err := c.MemoryEventFD() + fd, wd, err := c.MemoryEventFD() + + defer syscall.InotifyRmWatch(fd, wd) + defer syscall.Close(fd) + if err != nil { - errCh <- errors.Errorf("Failed to create memory event fd") + errCh <- err + return } + for { buffer := make([]byte, syscall.SizeofInotifyEvent*10) - bytesRead, err := syscall.Read(int(fd), buffer) + bytesRead, err := syscall.Read(fd, buffer) if err != nil { errCh <- err + return } var out map[string]interface{} if bytesRead >= syscall.SizeofInotifyEvent { @@ -572,6 +583,9 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { OOMKill: out["oom_kill"].(uint64), } ec <- e + } else { + errCh <- err + return } } } diff --git a/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go b/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go index d9b2bb0b6..9d5a1e774 100644 --- a/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go +++ b/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go @@ -25,14 +25,15 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package type Metrics struct { - Pids *PidsStat `protobuf:"bytes,1,opt,name=pids,proto3" json:"pids,omitempty"` - CPU *CPUStat `protobuf:"bytes,2,opt,name=cpu,proto3" json:"cpu,omitempty"` - Memory *MemoryStat `protobuf:"bytes,4,opt,name=memory,proto3" json:"memory,omitempty"` - Rdma *RdmaStat `protobuf:"bytes,5,opt,name=rdma,proto3" json:"rdma,omitempty"` - Io *IOStat `protobuf:"bytes,6,opt,name=io,proto3" json:"io,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Pids *PidsStat `protobuf:"bytes,1,opt,name=pids,proto3" json:"pids,omitempty"` + CPU *CPUStat `protobuf:"bytes,2,opt,name=cpu,proto3" json:"cpu,omitempty"` + Memory *MemoryStat `protobuf:"bytes,4,opt,name=memory,proto3" json:"memory,omitempty"` + Rdma *RdmaStat `protobuf:"bytes,5,opt,name=rdma,proto3" json:"rdma,omitempty"` + Io *IOStat `protobuf:"bytes,6,opt,name=io,proto3" json:"io,omitempty"` + Hugetlb []*HugeTlbStat `protobuf:"bytes,7,rep,name=hugetlb,proto3" json:"hugetlb,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Metrics) Reset() { *m = Metrics{} } @@ -388,6 +389,47 @@ func (m *IOEntry) XXX_DiscardUnknown() { var xxx_messageInfo_IOEntry proto.InternalMessageInfo +type HugeTlbStat struct { + Current uint64 `protobuf:"varint,1,opt,name=current,proto3" json:"current,omitempty"` + Max uint64 `protobuf:"varint,2,opt,name=max,proto3" json:"max,omitempty"` + Pagesize string `protobuf:"bytes,3,opt,name=pagesize,proto3" json:"pagesize,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HugeTlbStat) Reset() { *m = HugeTlbStat{} } +func (*HugeTlbStat) ProtoMessage() {} +func (*HugeTlbStat) Descriptor() ([]byte, []int) { + return fileDescriptor_2fc6005842049e6b, []int{8} +} +func (m *HugeTlbStat) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *HugeTlbStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_HugeTlbStat.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *HugeTlbStat) XXX_Merge(src proto.Message) { + xxx_messageInfo_HugeTlbStat.Merge(m, src) +} +func (m *HugeTlbStat) XXX_Size() int { + return m.Size() +} +func (m *HugeTlbStat) XXX_DiscardUnknown() { + xxx_messageInfo_HugeTlbStat.DiscardUnknown(m) +} + +var xxx_messageInfo_HugeTlbStat proto.InternalMessageInfo + func init() { proto.RegisterType((*Metrics)(nil), "io.containerd.cgroups.v2.Metrics") proto.RegisterType((*PidsStat)(nil), "io.containerd.cgroups.v2.PidsStat") @@ -397,6 +439,7 @@ func init() { proto.RegisterType((*RdmaEntry)(nil), "io.containerd.cgroups.v2.RdmaEntry") proto.RegisterType((*IOStat)(nil), "io.containerd.cgroups.v2.IOStat") proto.RegisterType((*IOEntry)(nil), "io.containerd.cgroups.v2.IOEntry") + proto.RegisterType((*HugeTlbStat)(nil), "io.containerd.cgroups.v2.HugeTlbStat") } func init() { @@ -404,74 +447,77 @@ func init() { } var fileDescriptor_2fc6005842049e6b = []byte{ - // 1064 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0x4f, 0x6f, 0x1c, 0xc5, - 0x13, 0xcd, 0xda, 0x9b, 0xfd, 0xd3, 0x6b, 0x27, 0x4e, 0xc7, 0xf1, 0xaf, 0xe3, 0xfc, 0xbc, 0xb6, - 0xd7, 0x80, 0x82, 0x04, 0xbb, 0xc8, 0xfc, 0x13, 0x28, 0x1c, 0x1c, 0x43, 0x04, 0x12, 0x26, 0xd6, - 0x26, 0x16, 0xc7, 0x51, 0xef, 0x4c, 0x7b, 0xb7, 0xed, 0x99, 0xe9, 0x51, 0x77, 0xaf, 0xad, 0xe5, - 0xc4, 0x81, 0x2b, 0xe2, 0x63, 0xf0, 0x55, 0x72, 0x83, 0x23, 0x27, 0x44, 0xfc, 0x49, 0x50, 0x55, - 0xf5, 0x78, 0x86, 0x83, 0x03, 0xb7, 0xae, 0xf7, 0x5e, 0xd5, 0x54, 0xbd, 0xd9, 0xae, 0x59, 0xf6, - 0xd1, 0x54, 0xfb, 0xd9, 0x7c, 0x32, 0x8c, 0x4d, 0x36, 0x8a, 0x4d, 0xee, 0xa5, 0xce, 0x95, 0x4d, - 0x46, 0xf1, 0xd4, 0x9a, 0x79, 0xe1, 0x46, 0x17, 0xfb, 0x23, 0xe7, 0xa5, 0x77, 0xa3, 0x4c, 0x79, - 0xab, 0x63, 0x37, 0x2c, 0xac, 0xf1, 0x86, 0x0b, 0x6d, 0x86, 0x95, 0x7a, 0x18, 0xd4, 0xc3, 0x8b, - 0xfd, 0xcd, 0xf5, 0xa9, 0x99, 0x1a, 0x14, 0x8d, 0xe0, 0x44, 0xfa, 0xc1, 0xaf, 0x4b, 0xac, 0x7d, - 0x44, 0x15, 0xf8, 0x27, 0xac, 0x59, 0xe8, 0xc4, 0x89, 0xc6, 0x4e, 0xe3, 0x71, 0x6f, 0x7f, 0x30, - 0xbc, 0xa9, 0xd4, 0xf0, 0x58, 0x27, 0xee, 0x85, 0x97, 0x7e, 0x8c, 0x7a, 0xfe, 0x84, 0x2d, 0xc7, - 0xc5, 0x5c, 0x2c, 0x61, 0xda, 0xee, 0xcd, 0x69, 0x87, 0xc7, 0x27, 0x90, 0xf5, 0xb4, 0x7d, 0xf5, - 0xe7, 0xf6, 0xf2, 0xe1, 0xf1, 0xc9, 0x18, 0xd2, 0xf8, 0x13, 0xd6, 0xca, 0x54, 0x66, 0xec, 0x42, - 0x34, 0xb1, 0xc0, 0x5b, 0x37, 0x17, 0x38, 0x42, 0x1d, 0x3e, 0x39, 0xe4, 0x40, 0xcf, 0x36, 0xc9, - 0xa4, 0xb8, 0xfd, 0x6f, 0x3d, 0x8f, 0x93, 0x4c, 0x52, 0xcf, 0xa0, 0xe7, 0x1f, 0xb0, 0x25, 0x6d, - 0x44, 0x0b, 0xb3, 0x76, 0x6e, 0xce, 0xfa, 0xe6, 0x39, 0xe6, 0x2c, 0x69, 0x33, 0xf8, 0x9c, 0x75, - 0xca, 0xb9, 0xb9, 0x60, 0xed, 0x78, 0x6e, 0xad, 0xca, 0x3d, 0x9a, 0xd5, 0x1c, 0x97, 0x21, 0x5f, - 0x67, 0xb7, 0x53, 0x9d, 0x69, 0x8f, 0x6e, 0x34, 0xc7, 0x14, 0x0c, 0x7e, 0x6b, 0xb0, 0x76, 0x98, - 0x9e, 0x6f, 0x31, 0x36, 0x77, 0x72, 0xaa, 0xa2, 0xb9, 0x53, 0x71, 0x48, 0xef, 0x22, 0x72, 0xe2, - 0x54, 0xcc, 0x1f, 0xb1, 0xee, 0xdc, 0x29, 0x4b, 0x2c, 0x15, 0xe9, 0x00, 0x80, 0xe4, 0x36, 0xeb, - 0xb9, 0x85, 0xf3, 0x2a, 0x23, 0x7a, 0x19, 0x69, 0x46, 0x10, 0x0a, 0xb6, 0x18, 0xcb, 0x6d, 0x54, - 0x28, 0xab, 0x4d, 0xe2, 0xd0, 0xd0, 0xe6, 0xb8, 0x9b, 0xdb, 0x63, 0x02, 0xf8, 0x2e, 0x5b, 0xc9, - 0x6d, 0xe4, 0x67, 0xd6, 0x78, 0x9f, 0xaa, 0x04, 0x5d, 0x6b, 0x8e, 0x7b, 0xb9, 0x7d, 0x59, 0x42, - 0xfc, 0x6d, 0x76, 0xe7, 0x9a, 0xa7, 0xa7, 0xb4, 0x50, 0xb4, 0x7a, 0x8d, 0xc2, 0x83, 0x06, 0xbf, - 0x74, 0x19, 0xab, 0x5e, 0x07, 0xe7, 0xac, 0x29, 0x73, 0x93, 0x87, 0x71, 0xf0, 0x0c, 0xd8, 0xa9, - 0x4e, 0x55, 0x18, 0x02, 0xcf, 0xd0, 0xc0, 0xb9, 0xb2, 0xb9, 0x4a, 0x23, 0xe7, 0x65, 0x7c, 0x1e, - 0x26, 0xe8, 0x11, 0xf6, 0x02, 0x20, 0x48, 0x73, 0xa9, 0x9c, 0x84, 0xe6, 0xf1, 0x8c, 0x98, 0x89, - 0xcf, 0x43, 0xbf, 0x78, 0x06, 0xa7, 0xdd, 0x2c, 0x53, 0x59, 0xe8, 0x8f, 0x02, 0x70, 0x08, 0x1e, - 0x14, 0x65, 0xb2, 0x28, 0x54, 0x22, 0xda, 0xe4, 0x10, 0x40, 0x47, 0x88, 0x80, 0x43, 0x28, 0x48, - 0xb4, 0xf5, 0x0b, 0xd1, 0x21, 0x87, 0x00, 0xf9, 0x12, 0x00, 0x18, 0x1f, 0xe9, 0x4b, 0xab, 0xbd, - 0x9a, 0x40, 0x8b, 0x5d, 0x1a, 0x1f, 0xd0, 0xef, 0x4b, 0x90, 0x3f, 0x64, 0x1d, 0x98, 0x31, 0xf2, - 0xb3, 0x42, 0x30, 0xfa, 0x05, 0x40, 0xfc, 0x72, 0x56, 0xf0, 0x3d, 0xb6, 0xaa, 0x73, 0x19, 0x7b, - 0x7d, 0xa1, 0x22, 0xf4, 0xa4, 0x87, 0xfc, 0x4a, 0x09, 0x1e, 0x80, 0x37, 0xdb, 0xac, 0x57, 0x97, - 0xac, 0x50, 0x9b, 0x35, 0x41, 0xbd, 0x0a, 0xba, 0xb8, 0xfa, 0xcf, 0x2a, 0xcf, 0xc0, 0xcd, 0xaa, - 0x0a, 0x4a, 0xee, 0xd4, 0xab, 0xa0, 0x60, 0x87, 0xf5, 0xe6, 0xb9, 0xba, 0xd0, 0xb1, 0x97, 0x93, - 0x54, 0x89, 0xbb, 0xe4, 0x76, 0x0d, 0xe2, 0xef, 0xb2, 0x35, 0x70, 0x38, 0xb2, 0x2a, 0x4e, 0xa5, - 0xce, 0x50, 0xb6, 0x86, 0xb2, 0xbb, 0x80, 0x8f, 0x2b, 0x98, 0xbf, 0xcf, 0x38, 0x4a, 0xe7, 0x79, - 0x5d, 0x7c, 0x0f, 0xc5, 0xf7, 0x80, 0x39, 0xa9, 0x13, 0x70, 0x47, 0x8a, 0xe9, 0xa9, 0x9c, 0xa7, - 0x5e, 0x70, 0x72, 0x28, 0x84, 0xbc, 0xcf, 0x58, 0x31, 0xcd, 0xe4, 0x19, 0x91, 0xf7, 0xa9, 0xeb, - 0x0a, 0x81, 0x07, 0x5d, 0x1a, 0x7b, 0xae, 0xf3, 0xa9, 0x53, 0x3e, 0xb2, 0x8a, 0x74, 0xeb, 0xf4, - 0xa0, 0x8a, 0x19, 0x13, 0xc1, 0x47, 0xec, 0x7e, 0x4d, 0x8e, 0xd3, 0x4b, 0xaf, 0xc4, 0x03, 0xd4, - 0xd7, 0x2a, 0x1d, 0x04, 0x86, 0x7f, 0xcc, 0x36, 0x6a, 0x09, 0xb9, 0x49, 0x54, 0xe8, 0x5b, 0x6c, - 0x60, 0xce, 0x83, 0x8a, 0xfd, 0xae, 0x22, 0xf9, 0x26, 0xeb, 0x14, 0x53, 0xab, 0x4e, 0x75, 0x9a, - 0x8a, 0xff, 0xd1, 0xc5, 0x2c, 0x63, 0xbe, 0xc1, 0x5a, 0xc5, 0xd4, 0xc5, 0x32, 0x17, 0x02, 0x99, - 0x10, 0x91, 0x09, 0xce, 0x2b, 0x99, 0x8a, 0x87, 0xa5, 0x09, 0x18, 0x92, 0x09, 0xd7, 0xcd, 0x6e, - 0x96, 0x26, 0x94, 0x08, 0x1f, 0xb0, 0x95, 0x62, 0x9a, 0xa8, 0x6b, 0xc5, 0x23, 0x7a, 0xff, 0x75, - 0x8c, 0x6a, 0xa4, 0xf2, 0x87, 0xc5, 0xa9, 0x55, 0x4a, 0xfc, 0xbf, 0xac, 0x51, 0x22, 0xf0, 0xfa, - 0xab, 0x28, 0x11, 0x5b, 0xf4, 0xfa, 0x6b, 0x10, 0x7f, 0x87, 0xdd, 0xf5, 0xb3, 0x22, 0x42, 0x23, - 0x23, 0x99, 0xa6, 0x26, 0x16, 0xfd, 0xf2, 0xba, 0x17, 0xcf, 0x00, 0x3d, 0x00, 0x90, 0xbf, 0xc7, - 0x38, 0xe8, 0x62, 0x93, 0xa6, 0xb2, 0x70, 0x2a, 0x48, 0xb7, 0x51, 0xba, 0xe6, 0x67, 0xc5, 0x61, - 0x20, 0x48, 0xbd, 0xce, 0x6e, 0xe3, 0x42, 0x13, 0x3b, 0x74, 0x35, 0x31, 0x80, 0x5f, 0x2b, 0x2d, - 0x3e, 0x5a, 0x90, 0xbb, 0xd4, 0x2e, 0x42, 0xdf, 0x02, 0x02, 0x57, 0xd3, 0x5d, 0xca, 0x22, 0xa2, - 0xdc, 0x01, 0x5d, 0x4d, 0x40, 0x4e, 0x30, 0xbf, 0xa4, 0x29, 0x7d, 0xaf, 0xa2, 0x31, 0x7b, 0xf0, - 0x53, 0x83, 0x75, 0xca, 0x25, 0xcf, 0xbf, 0xa8, 0x2f, 0xe8, 0xe5, 0xc7, 0xbd, 0xfd, 0xbd, 0x37, - 0x7f, 0x19, 0xbe, 0xca, 0xbd, 0x5d, 0x54, 0x5b, 0xfc, 0xb3, 0x6a, 0x8b, 0xff, 0xe7, 0xe4, 0xb0, - 0xea, 0x15, 0xeb, 0x5e, 0x63, 0xf0, 0xb3, 0x48, 0xe0, 0xae, 0x29, 0x5c, 0x8c, 0xdd, 0x71, 0x88, - 0xc0, 0x8a, 0x59, 0x2c, 0xa3, 0x99, 0xcc, 0x93, 0x54, 0x39, 0xdc, 0x90, 0xab, 0x63, 0x36, 0x8b, - 0xe5, 0xd7, 0x84, 0x94, 0x02, 0x33, 0x39, 0x53, 0xb1, 0x77, 0xb8, 0x26, 0x49, 0xf0, 0x9c, 0x90, - 0xc1, 0x01, 0x6b, 0xd1, 0xb7, 0x89, 0x7f, 0x5a, 0x9a, 0x4d, 0x83, 0xee, 0xbe, 0xe9, 0x63, 0x16, - 0x3a, 0x45, 0xfd, 0xe0, 0xe7, 0x06, 0x6b, 0x07, 0x08, 0xde, 0x58, 0x26, 0xcf, 0x8c, 0x0d, 0x0b, - 0x9c, 0x02, 0x44, 0x75, 0x6e, 0x6c, 0xf9, 0x31, 0xc3, 0x00, 0x86, 0xb2, 0x93, 0x85, 0x57, 0x2e, - 0x6c, 0xef, 0x10, 0x01, 0x7e, 0x49, 0x38, 0xad, 0xee, 0x10, 0xc1, 0xf2, 0xb6, 0xda, 0xb8, 0x72, - 0x79, 0xc3, 0x19, 0xb0, 0x4b, 0xc0, 0x68, 0x77, 0xe3, 0xf9, 0xa9, 0x78, 0xf5, 0xba, 0x7f, 0xeb, - 0x8f, 0xd7, 0xfd, 0x5b, 0x3f, 0x5e, 0xf5, 0x1b, 0xaf, 0xae, 0xfa, 0x8d, 0xdf, 0xaf, 0xfa, 0x8d, - 0xbf, 0xae, 0xfa, 0x8d, 0x49, 0x0b, 0xff, 0xab, 0x7c, 0xf8, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xf0, 0xd3, 0x07, 0x15, 0x13, 0x09, 0x00, 0x00, + // 1118 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0x4d, 0x6f, 0x1c, 0x45, + 0x10, 0xcd, 0xda, 0x9b, 0xfd, 0xe8, 0xb5, 0x13, 0xa7, 0xe3, 0x84, 0x4e, 0x42, 0xd6, 0xf6, 0x86, + 0xa0, 0x20, 0xc1, 0x2e, 0x32, 0x5f, 0x02, 0x05, 0x21, 0x27, 0x10, 0x05, 0x09, 0x13, 0x6b, 0xe2, + 0x15, 0xc7, 0x51, 0xef, 0x4c, 0x7b, 0x76, 0xec, 0xf9, 0x52, 0x77, 0x8f, 0xcd, 0xe6, 0xc4, 0x81, + 0x2b, 0xe2, 0x6f, 0xe5, 0x06, 0x47, 0x4e, 0x88, 0xf8, 0xc4, 0xcf, 0x40, 0x55, 0xd5, 0xb3, 0x33, + 0x1c, 0x6c, 0xb8, 0x75, 0xbd, 0x7a, 0xaf, 0xa6, 0xfa, 0xf5, 0x76, 0xf5, 0xb2, 0x8f, 0xa3, 0xd8, + 0xce, 0xcb, 0xd9, 0x38, 0xc8, 0xd3, 0x49, 0x90, 0x67, 0x56, 0xc6, 0x99, 0xd2, 0xe1, 0x24, 0x88, + 0x74, 0x5e, 0x16, 0x66, 0x72, 0xba, 0x3b, 0x31, 0x56, 0x5a, 0x33, 0x49, 0x95, 0xd5, 0x71, 0x60, + 0xc6, 0x85, 0xce, 0x6d, 0xce, 0x45, 0x9c, 0x8f, 0x6b, 0xf6, 0xd8, 0xb1, 0xc7, 0xa7, 0xbb, 0x77, + 0x37, 0xa3, 0x3c, 0xca, 0x91, 0x34, 0x81, 0x15, 0xf1, 0x47, 0x7f, 0xaf, 0xb0, 0xee, 0x3e, 0x55, + 0xe0, 0x9f, 0xb2, 0x76, 0x11, 0x87, 0x46, 0xb4, 0xb6, 0x5b, 0x8f, 0x06, 0xbb, 0xa3, 0xf1, 0x45, + 0xa5, 0xc6, 0x07, 0x71, 0x68, 0x5e, 0x5a, 0x69, 0x3d, 0xe4, 0xf3, 0xc7, 0x6c, 0x35, 0x28, 0x4a, + 0xb1, 0x82, 0xb2, 0x9d, 0x8b, 0x65, 0x4f, 0x0f, 0xa6, 0xa0, 0x7a, 0xd2, 0x3d, 0xff, 0x73, 0x6b, + 0xf5, 0xe9, 0xc1, 0xd4, 0x03, 0x19, 0x7f, 0xcc, 0x3a, 0xa9, 0x4a, 0x73, 0xbd, 0x10, 0x6d, 0x2c, + 0xf0, 0xce, 0xc5, 0x05, 0xf6, 0x91, 0x87, 0x5f, 0x76, 0x1a, 0xe8, 0x59, 0x87, 0xa9, 0x14, 0x57, + 0xff, 0xab, 0x67, 0x2f, 0x4c, 0x25, 0xf5, 0x0c, 0x7c, 0xfe, 0x21, 0x5b, 0x89, 0x73, 0xd1, 0x41, + 0xd5, 0xf6, 0xc5, 0xaa, 0x6f, 0x5f, 0xa0, 0x66, 0x25, 0xce, 0xf9, 0x57, 0xac, 0x3b, 0x2f, 0x23, + 0x65, 0x93, 0x99, 0xe8, 0x6e, 0xaf, 0x3e, 0x1a, 0xec, 0x3e, 0xbc, 0x58, 0xf6, 0xbc, 0x8c, 0xd4, + 0x61, 0x32, 0x43, 0x6d, 0xa5, 0x1a, 0x7d, 0xc1, 0x7a, 0x95, 0x71, 0x5c, 0xb0, 0x6e, 0x50, 0x6a, + 0xad, 0x32, 0x8b, 0x6e, 0xb7, 0xbd, 0x2a, 0xe4, 0x9b, 0xec, 0x6a, 0x12, 0xa7, 0xb1, 0x45, 0x3b, + 0xdb, 0x1e, 0x05, 0xa3, 0xdf, 0x5a, 0xac, 0xeb, 0xec, 0xe3, 0xf7, 0x19, 0x2b, 0x8d, 0x8c, 0x94, + 0x5f, 0x1a, 0x15, 0x38, 0x79, 0x1f, 0x91, 0xa9, 0x51, 0x01, 0xbf, 0xc7, 0xfa, 0xa5, 0x51, 0x9a, + 0xb2, 0x54, 0xa4, 0x07, 0x00, 0x26, 0xb7, 0xd8, 0xc0, 0x2c, 0x8c, 0x55, 0x29, 0xa5, 0x57, 0x31, + 0xcd, 0x08, 0x42, 0xc2, 0x7d, 0xc6, 0x32, 0xed, 0x17, 0x4a, 0xc7, 0x79, 0x68, 0xf0, 0x44, 0xda, + 0x5e, 0x3f, 0xd3, 0x07, 0x04, 0xf0, 0x1d, 0xb6, 0x96, 0x69, 0xdf, 0xce, 0x75, 0x6e, 0x6d, 0xa2, + 0x42, 0xb4, 0xbd, 0xed, 0x0d, 0x32, 0x7d, 0x58, 0x41, 0xfc, 0x21, 0xbb, 0xb6, 0xcc, 0xd3, 0x57, + 0x3a, 0x48, 0x5a, 0x5f, 0xa2, 0xf0, 0xa1, 0xd1, 0xaf, 0x7d, 0xc6, 0xea, 0xf3, 0xe4, 0x9c, 0xb5, + 0x65, 0x96, 0x67, 0x6e, 0x3b, 0xb8, 0x06, 0xec, 0x28, 0x4e, 0x94, 0xdb, 0x04, 0xae, 0xa1, 0x81, + 0x13, 0xa5, 0x33, 0x95, 0xf8, 0xc6, 0xca, 0xe0, 0xc4, 0xed, 0x60, 0x40, 0xd8, 0x4b, 0x80, 0x40, + 0x66, 0x12, 0x39, 0x73, 0xcd, 0xe3, 0x1a, 0xb1, 0x3c, 0x38, 0x71, 0xfd, 0xe2, 0x1a, 0x9c, 0x36, + 0xf3, 0x54, 0xa5, 0xae, 0x3f, 0x0a, 0xc0, 0x21, 0xf8, 0x90, 0x9f, 0xca, 0xa2, 0x50, 0xa1, 0xe8, + 0x92, 0x43, 0x00, 0xed, 0x23, 0x02, 0x0e, 0x21, 0x21, 0x8c, 0xb5, 0x5d, 0x88, 0x1e, 0x39, 0x04, + 0xc8, 0xd7, 0x00, 0xc0, 0xf6, 0x31, 0x7d, 0xa6, 0x63, 0xab, 0x66, 0xd0, 0x62, 0x9f, 0xb6, 0x0f, + 0xe8, 0x0f, 0x15, 0xc8, 0xef, 0xb0, 0x1e, 0xec, 0xd1, 0xb7, 0xf3, 0x42, 0x30, 0xfa, 0x05, 0x40, + 0x7c, 0x38, 0x2f, 0xf8, 0x03, 0xb6, 0x1e, 0x67, 0x32, 0xb0, 0xf1, 0xa9, 0xf2, 0xd1, 0x93, 0x01, + 0xe6, 0xd7, 0x2a, 0x70, 0x0f, 0xbc, 0xd9, 0x62, 0x83, 0x26, 0x65, 0x8d, 0xda, 0x6c, 0x10, 0x9a, + 0x55, 0xd0, 0xc5, 0xf5, 0x7f, 0x57, 0x79, 0x06, 0x6e, 0xd6, 0x55, 0x90, 0x72, 0xad, 0x59, 0x05, + 0x09, 0xdb, 0x6c, 0x50, 0x66, 0xea, 0x34, 0x0e, 0xac, 0x9c, 0x25, 0x4a, 0x5c, 0x27, 0xb7, 0x1b, + 0x10, 0x7f, 0x8f, 0x6d, 0x80, 0xc3, 0xbe, 0x56, 0x41, 0x22, 0xe3, 0x14, 0x69, 0x1b, 0x48, 0xbb, + 0x0e, 0xb8, 0x57, 0xc3, 0xfc, 0x03, 0xc6, 0x91, 0x5a, 0x66, 0x4d, 0xf2, 0x0d, 0x24, 0xdf, 0x80, + 0xcc, 0xb4, 0x99, 0x80, 0x3b, 0x52, 0x44, 0x47, 0xb2, 0x4c, 0xac, 0xe0, 0xe4, 0x90, 0x0b, 0xf9, + 0x90, 0xb1, 0x22, 0x4a, 0xe5, 0x31, 0x25, 0x6f, 0x52, 0xd7, 0x35, 0x02, 0x1f, 0x3a, 0xcb, 0xf5, + 0x49, 0x9c, 0x45, 0x46, 0x59, 0x5f, 0x2b, 0xe2, 0x6d, 0xd2, 0x87, 0xea, 0x8c, 0x47, 0x09, 0x3e, + 0x61, 0x37, 0x1b, 0x74, 0xdc, 0xbd, 0xb4, 0x4a, 0xdc, 0x42, 0x7e, 0xa3, 0xd2, 0x9e, 0xcb, 0xf0, + 0x4f, 0xd8, 0xed, 0x86, 0x20, 0xcb, 0x43, 0xe5, 0xfa, 0x16, 0xb7, 0x51, 0x73, 0xab, 0xce, 0x7e, + 0x5f, 0x27, 0xf9, 0x5d, 0xd6, 0x2b, 0x22, 0xad, 0x8e, 0xe2, 0x24, 0x11, 0x6f, 0xd1, 0xc5, 0xac, + 0x62, 0x7e, 0x9b, 0x75, 0x8a, 0xc8, 0x04, 0x32, 0x13, 0x02, 0x33, 0x2e, 0x22, 0x13, 0x8c, 0x55, + 0x32, 0x11, 0x77, 0x2a, 0x13, 0x30, 0x24, 0x13, 0x96, 0xcd, 0xde, 0xad, 0x4c, 0xa8, 0x10, 0x3e, + 0x62, 0x6b, 0x45, 0x14, 0xaa, 0x25, 0xe3, 0x1e, 0x9d, 0x7f, 0x13, 0xa3, 0x1a, 0x89, 0x7c, 0xb5, + 0x38, 0xd2, 0x4a, 0x89, 0xb7, 0xab, 0x1a, 0x15, 0x02, 0xc7, 0x5f, 0x47, 0xa1, 0xb8, 0x4f, 0xc7, + 0xdf, 0x80, 0xf8, 0xbb, 0xec, 0xba, 0x9d, 0x17, 0x3e, 0x1a, 0xe9, 0xcb, 0x24, 0xc9, 0x03, 0x31, + 0xac, 0xae, 0x7b, 0xf1, 0x0c, 0xd0, 0x3d, 0x00, 0xf9, 0xfb, 0x8c, 0x03, 0x2f, 0xc8, 0x93, 0x44, + 0x16, 0x46, 0x39, 0xea, 0x16, 0x52, 0x37, 0xec, 0xbc, 0x78, 0xea, 0x12, 0xc4, 0xde, 0x64, 0x57, + 0x71, 0xa0, 0x89, 0x6d, 0xba, 0x9a, 0x18, 0xc0, 0xaf, 0x95, 0x06, 0x1f, 0x0d, 0xc8, 0x1d, 0x6a, + 0x17, 0xa1, 0xef, 0x00, 0x81, 0xab, 0x69, 0xce, 0x64, 0xe1, 0x93, 0x76, 0x44, 0x57, 0x13, 0x90, + 0x29, 0xea, 0xab, 0x34, 0xc9, 0x1f, 0xd4, 0x69, 0x54, 0x8f, 0x7e, 0x6e, 0xb1, 0x5e, 0xf5, 0x4a, + 0xf0, 0x2f, 0x9b, 0x03, 0x1a, 0xa6, 0xfd, 0x83, 0xcb, 0x9f, 0x96, 0x6f, 0x32, 0xab, 0x17, 0xf5, + 0x14, 0xff, 0xbc, 0x9e, 0xe2, 0xff, 0x5b, 0xec, 0x46, 0xbd, 0x62, 0xfd, 0x25, 0x06, 0x3f, 0x8b, + 0x10, 0xee, 0x9a, 0xc2, 0xc1, 0xd8, 0xf7, 0x5c, 0x04, 0x56, 0xcc, 0x03, 0xe9, 0xcf, 0x65, 0x16, + 0x26, 0xca, 0xe0, 0x84, 0x5c, 0xf7, 0xd8, 0x3c, 0x90, 0xcf, 0x09, 0xa9, 0x08, 0xf9, 0xec, 0x58, + 0x05, 0xd6, 0xe0, 0x98, 0x24, 0xc2, 0x0b, 0x42, 0x46, 0x7b, 0xac, 0x43, 0x8f, 0x1b, 0xff, 0xac, + 0x32, 0x9b, 0x36, 0xba, 0x73, 0xd9, 0x6b, 0xe8, 0x3a, 0x45, 0xfe, 0xe8, 0x97, 0x16, 0xeb, 0x3a, + 0x08, 0x4e, 0x2c, 0x95, 0xc7, 0xb9, 0x76, 0x03, 0x9c, 0x02, 0x44, 0xe3, 0x2c, 0xd7, 0xd5, 0x63, + 0x86, 0x01, 0x6c, 0x4a, 0xcf, 0x16, 0x56, 0x19, 0x37, 0xbd, 0x5d, 0x04, 0xf8, 0x19, 0xe1, 0x34, + 0xba, 0x5d, 0x04, 0xc3, 0x5b, 0xc7, 0xb9, 0xa9, 0x86, 0x37, 0xac, 0x01, 0x3b, 0x03, 0x8c, 0x66, + 0x37, 0xae, 0x47, 0x53, 0x36, 0x68, 0x3c, 0xbc, 0x97, 0xbc, 0xb1, 0x1b, 0x6c, 0x35, 0x95, 0x3f, + 0xba, 0xa6, 0x60, 0x89, 0x57, 0x53, 0x46, 0xca, 0xc4, 0xaf, 0x14, 0x36, 0xd5, 0xf7, 0x96, 0xf1, + 0x13, 0xf1, 0xfa, 0xcd, 0xf0, 0xca, 0x1f, 0x6f, 0x86, 0x57, 0x7e, 0x3a, 0x1f, 0xb6, 0x5e, 0x9f, + 0x0f, 0x5b, 0xbf, 0x9f, 0x0f, 0x5b, 0x7f, 0x9d, 0x0f, 0x5b, 0xb3, 0x0e, 0xfe, 0x87, 0xfa, 0xe8, + 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x79, 0xd2, 0xcd, 0xe2, 0xab, 0x09, 0x00, 0x00, } func (m *Metrics) Marshal() (dAtA []byte, err error) { @@ -539,6 +585,18 @@ func (m *Metrics) MarshalTo(dAtA []byte) (int, error) { } i += n5 } + if len(m.Hugetlb) > 0 { + for _, msg := range m.Hugetlb { + dAtA[i] = 0x3a + i++ + i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -1029,6 +1087,43 @@ func (m *IOEntry) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *HugeTlbStat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *HugeTlbStat) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Current != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + } + if m.Max != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + } + if len(m.Pagesize) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) + i += copy(dAtA[i:], m.Pagesize) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintMetrics(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -1064,6 +1159,12 @@ func (m *Metrics) Size() (n int) { l = m.Io.Size() n += 1 + l + sovMetrics(uint64(l)) } + if len(m.Hugetlb) > 0 { + for _, e := range m.Hugetlb { + l = e.Size() + n += 1 + l + sovMetrics(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -1329,6 +1430,28 @@ func (m *IOEntry) Size() (n int) { return n } +func (m *HugeTlbStat) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Current != 0 { + n += 1 + sovMetrics(uint64(m.Current)) + } + if m.Max != 0 { + n += 1 + sovMetrics(uint64(m.Max)) + } + l = len(m.Pagesize) + if l > 0 { + n += 1 + l + sovMetrics(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovMetrics(x uint64) (n int) { for { n++ @@ -1352,6 +1475,7 @@ func (this *Metrics) String() string { `Memory:` + strings.Replace(fmt.Sprintf("%v", this.Memory), "MemoryStat", "MemoryStat", 1) + `,`, `Rdma:` + strings.Replace(fmt.Sprintf("%v", this.Rdma), "RdmaStat", "RdmaStat", 1) + `,`, `Io:` + strings.Replace(fmt.Sprintf("%v", this.Io), "IOStat", "IOStat", 1) + `,`, + `Hugetlb:` + strings.Replace(fmt.Sprintf("%v", this.Hugetlb), "HugeTlbStat", "HugeTlbStat", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1482,6 +1606,19 @@ func (this *IOEntry) String() string { }, "") return s } +func (this *HugeTlbStat) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&HugeTlbStat{`, + `Current:` + fmt.Sprintf("%v", this.Current) + `,`, + `Max:` + fmt.Sprintf("%v", this.Max) + `,`, + `Pagesize:` + fmt.Sprintf("%v", this.Pagesize) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func valueToStringMetrics(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -1699,6 +1836,40 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hugetlb", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMetrics + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMetrics + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hugetlb = append(m.Hugetlb, &HugeTlbStat{}) + if err := m.Hugetlb[len(m.Hugetlb)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMetrics(dAtA[iNdEx:]) @@ -3205,6 +3376,130 @@ func (m *IOEntry) Unmarshal(dAtA []byte) error { } return nil } +func (m *HugeTlbStat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: HugeTlbStat: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: HugeTlbStat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) + } + m.Current = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Current |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pagesize", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetrics + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetrics + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pagesize = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMetrics(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMetrics + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMetrics + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipMetrics(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/vendor/github.com/containerd/cgroups/v2/stats/metrics.proto b/vendor/github.com/containerd/cgroups/v2/stats/metrics.proto index 468656391..4a8535399 100644 --- a/vendor/github.com/containerd/cgroups/v2/stats/metrics.proto +++ b/vendor/github.com/containerd/cgroups/v2/stats/metrics.proto @@ -10,6 +10,7 @@ message Metrics { MemoryStat memory = 4; RdmaStat rdma = 5; IOStat io = 6; + repeated HugeTlbStat hugetlb = 7; } message PidsStat { @@ -87,3 +88,9 @@ message IOEntry { uint64 rios = 5; uint64 wios = 6; } + +message HugeTlbStat { + uint64 current = 1; + uint64 max = 2; + string pagesize = 3; +} diff --git a/vendor/github.com/containerd/cgroups/v2/utils.go b/vendor/github.com/containerd/cgroups/v2/utils.go index 5de5414e4..d20c15cd6 100644 --- a/vendor/github.com/containerd/cgroups/v2/utils.go +++ b/vendor/github.com/containerd/cgroups/v2/utils.go @@ -85,6 +85,9 @@ func parseCgroupProcsFile(path string) ([]uint64, error) { out = append(out, pid) } } + if err := s.Err(); err != nil { + return nil, err + } return out, nil } @@ -144,9 +147,6 @@ func parseCgroupFromReader(r io.Reader) (string, error) { s = bufio.NewScanner(r) ) for s.Scan() { - if err := s.Err(); err != nil { - return "", err - } var ( text = s.Text() parts = strings.SplitN(text, ":", 3) @@ -159,6 +159,9 @@ func parseCgroupFromReader(r io.Reader) (string, error) { return parts[2], nil } } + if err := s.Err(); err != nil { + return "", err + } return "", fmt.Errorf("cgroup path not found") } @@ -194,6 +197,16 @@ func ToResources(spec *specs.LinuxResources) *Resources { resources.Memory.Low = l } } + if hugetlbs := spec.HugepageLimits; hugetlbs != nil { + hugeTlbUsage := HugeTlb{} + for _, hugetlb := range hugetlbs { + hugeTlbUsage = append(hugeTlbUsage, HugeTlbEntry{ + HugePageSize: hugetlb.Pagesize, + Limit: hugetlb.Limit, + }) + } + resources.HugeTlb = &hugeTlbUsage + } if pids := spec.Pids; pids != nil { resources.Pids = &Pids{ Max: pids.Limit, @@ -202,7 +215,7 @@ func ToResources(spec *specs.LinuxResources) *Resources { if i := spec.BlockIO; i != nil { resources.IO = &IO{} if i.Weight != nil { - resources.IO.BFQ.Weight = *i.Weight + resources.IO.BFQ.Weight = 1 + (*i.Weight-10)*9999/990 } for t, devices := range map[IOType][]specs.LinuxThrottleDevice{ ReadBPS: i.ThrottleReadBpsDevice, @@ -377,3 +390,56 @@ func systemdUnitFromPath(path string) string { _, unit := filepath.Split(path) return unit } + +func readHugeTlbStats(path string) []*stats.HugeTlbStat { + var usage = []*stats.HugeTlbStat{} + var keyUsage = make(map[string]*stats.HugeTlbStat) + f, err := os.Open(path) + if err != nil { + return usage + } + files, err := f.Readdir(-1) + f.Close() + if err != nil { + return usage + } + + for _, file := range files { + if strings.Contains(file.Name(), "hugetlb") && + (strings.HasSuffix(file.Name(), "max") || strings.HasSuffix(file.Name(), "current")) { + var hugeTlb *stats.HugeTlbStat + var ok bool + fileName := strings.Split(file.Name(), ".") + pageSize := fileName[1] + if hugeTlb, ok = keyUsage[pageSize]; !ok { + hugeTlb = &stats.HugeTlbStat{} + } + hugeTlb.Pagesize = pageSize + out, err := ioutil.ReadFile(filepath.Join(path, file.Name())) + if err != nil { + continue + } + var value uint64 + stringVal := strings.TrimSpace(string(out)) + if stringVal == "max" { + value = math.MaxUint64 + } else { + value, err = strconv.ParseUint(stringVal, 10, 64) + } + if err != nil { + continue + } + switch fileName[2] { + case "max": + hugeTlb.Max = value + case "current": + hugeTlb.Current = value + } + keyUsage[pageSize] = hugeTlb + } + } + for _, entry := range keyUsage { + usage = append(usage, entry) + } + return usage +}