deps: Bump cAdvisor to v0.46.0

Signed-off-by: David Porter <david@porter.me>
This commit is contained in:
David Porter
2022-11-08 18:49:14 -08:00
parent e62cfabf93
commit a2c4672163
46 changed files with 447 additions and 265 deletions

View File

@@ -9,6 +9,3 @@
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/

View File

@@ -1,24 +0,0 @@
dist: bionic
language: go
go:
- "1.13.x"
- "1.15.x"
install:
# Don't change local go.{mod, sum} by go get tools.
#
# ref: https://github.com/golang/go/issues/27643
- pushd ..; go get -u github.com/vbatts/git-validation; popd
- pushd ..; go get -u github.com/kunalkushwaha/ltag; popd
before_script:
- pushd ..; git clone https://github.com/containerd/project; popd
script:
- DCO_VERBOSITY=-q ../project/script/validate/dco
- ../project/script/validate/fileheader ../project/
- go test -v -race -covermode=atomic -coverprofile=coverage.txt ./...
after_success:
- bash <(curl -s https://codecov.io/bash)

View File

@@ -1,6 +1,7 @@
# ttrpc
[![Build Status](https://travis-ci.org/containerd/ttrpc.svg?branch=master)](https://travis-ci.org/containerd/ttrpc)
[![Build Status](https://github.com/containerd/ttrpc/workflows/CI/badge.svg)](https://github.com/containerd/ttrpc/actions?query=workflow%3ACI)
[![codecov](https://codecov.io/gh/containerd/ttrpc/branch/main/graph/badge.svg)](https://codecov.io/gh/containerd/ttrpc)
GRPC for low-memory environments.
@@ -40,13 +41,8 @@ directly, if required.
# Status
Very new. YMMV.
TODO:
- [X] Plumb error codes and GRPC status
- [X] Remove use of any type and dependency on typeurl package
- [X] Ensure that protocol can support streaming in the future
- [ ] Document protocol layout
- [ ] Add testing under concurrent load to ensure
- [ ] Verify connection error handling
@@ -55,8 +51,8 @@ TODO:
ttrpc is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE).
As a containerd sub-project, you will find the:
* [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
* [Project governance](https://github.com/containerd/project/blob/main/GOVERNANCE.md),
* [Maintainers](https://github.com/containerd/project/blob/main/MAINTAINERS),
* and [Contributing guidelines](https://github.com/containerd/project/blob/main/CONTRIBUTING.md)
information in our [`containerd/project`](https://github.com/containerd/project) repository.

View File

@@ -19,11 +19,11 @@ package ttrpc
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"net"
"sync"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -105,7 +105,7 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
if mh.Length > uint32(messageLengthMax) {
if _, err := ch.br.Discard(int(mh.Length)); err != nil {
return mh, nil, errors.Wrapf(err, "failed to discard after receiving oversized message")
return mh, nil, fmt.Errorf("failed to discard after receiving oversized message: %w", err)
}
return mh, nil, status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", mh.Length, messageLengthMax)
@@ -113,7 +113,7 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
p := ch.getmbuf(int(mh.Length))
if _, err := io.ReadFull(ch.br, p); err != nil {
return messageHeader{}, nil, errors.Wrapf(err, "failed reading message")
return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
}
return mh, p, nil

View File

@@ -18,6 +18,7 @@ package ttrpc
import (
"context"
"errors"
"io"
"net"
"os"
@@ -27,7 +28,6 @@ import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -194,72 +194,131 @@ type message struct {
err error
}
type receiver struct {
wg *sync.WaitGroup
messages chan *message
err error
// callMap provides access to a map of active calls, guarded by a mutex.
type callMap struct {
m sync.Mutex
activeCalls map[uint32]*callRequest
closeErr error
}
func (r *receiver) run(ctx context.Context, c *channel) {
defer r.wg.Done()
for {
select {
case <-ctx.Done():
r.err = ctx.Err()
return
default:
mh, p, err := c.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
r.err = filterCloseErr(err)
return
}
}
select {
case r.messages <- &message{
messageHeader: mh,
p: p[:mh.Length],
err: err,
}:
case <-ctx.Done():
r.err = ctx.Err()
return
}
}
// newCallMap returns a new callMap with an empty set of active calls.
func newCallMap() *callMap {
return &callMap{
activeCalls: make(map[uint32]*callRequest),
}
}
// set adds a call entry to the map with the given streamID key.
func (cm *callMap) set(streamID uint32, cr *callRequest) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
cm.activeCalls[streamID] = cr
return nil
}
// get looks up the call entry for the given streamID key, then removes it
// from the map and returns it.
func (cm *callMap) get(streamID uint32) (cr *callRequest, ok bool, err error) {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return nil, false, cm.closeErr
}
cr, ok = cm.activeCalls[streamID]
if ok {
delete(cm.activeCalls, streamID)
}
return
}
// abort sends the given error to each active call, and clears the map.
// Once abort has been called, any subsequent calls to the callMap will return the error passed to abort.
func (cm *callMap) abort(err error) error {
cm.m.Lock()
defer cm.m.Unlock()
if cm.closeErr != nil {
return cm.closeErr
}
for streamID, call := range cm.activeCalls {
call.errs <- err
delete(cm.activeCalls, streamID)
}
cm.closeErr = err
return nil
}
func (c *Client) run() {
var (
streamID uint32 = 1
waiters = make(map[uint32]*callRequest)
calls = c.calls
incoming = make(chan *message)
receiversDone = make(chan struct{})
wg sync.WaitGroup
waiters = newCallMap()
receiverDone = make(chan struct{})
)
// broadcast the shutdown error to the remaining waiters.
abortWaiters := func(wErr error) {
for _, waiter := range waiters {
waiter.errs <- wErr
}
}
recv := &receiver{
wg: &wg,
messages: incoming,
}
wg.Add(1)
// Sender goroutine
// Receives calls from dispatch, adds them to the set of active calls, and sends them
// to the server.
go func() {
wg.Wait()
close(receiversDone)
var streamID uint32 = 1
for {
select {
case <-c.ctx.Done():
return
case call := <-c.calls:
id := streamID
streamID += 2 // enforce odd client initiated request ids
if err := waiters.set(id, call); err != nil {
call.errs <- err // errs is buffered so should not block.
continue
}
if err := c.send(id, messageTypeRequest, call.req); err != nil {
call.errs <- err // errs is buffered so should not block.
waiters.get(id) // remove from waiters set
}
}
}
}()
// Receiver goroutine
// Receives responses from the server, looks up the call info in the set of active calls,
// and notifies the caller of the response.
go func() {
defer close(receiverDone)
for {
select {
case <-c.ctx.Done():
c.setError(c.ctx.Err())
return
default:
mh, p, err := c.channel.recv()
if err != nil {
_, ok := status.FromError(err)
if !ok {
// treat all errors that are not an rpc status as terminal.
// all others poison the connection.
c.setError(filterCloseErr(err))
return
}
}
msg := &message{
messageHeader: mh,
p: p[:mh.Length],
err: err,
}
call, ok, err := waiters.get(mh.StreamID)
if err != nil {
logrus.Errorf("ttrpc: failed to look up active call: %s", err)
continue
}
if !ok {
logrus.Errorf("ttrpc: received message for unknown channel %v", mh.StreamID)
continue
}
call.errs <- c.recv(call.resp, msg)
}
}
}()
go recv.run(c.ctx, c.channel)
defer func() {
c.conn.Close()
@@ -269,32 +328,14 @@ func (c *Client) run() {
for {
select {
case call := <-calls:
if err := c.send(streamID, messageTypeRequest, call.req); err != nil {
call.errs <- err
continue
}
waiters[streamID] = call
streamID += 2 // enforce odd client initiated request ids
case msg := <-incoming:
call, ok := waiters[msg.StreamID]
if !ok {
logrus.Errorf("ttrpc: received message for unknown channel %v", msg.StreamID)
continue
}
call.errs <- c.recv(call.resp, msg)
delete(waiters, msg.StreamID)
case <-receiversDone:
// all the receivers have exited
if recv.err != nil {
c.setError(recv.err)
}
case <-receiverDone:
// The receiver has exited.
// don't return out, let the close of the context trigger the abort of waiters
c.Close()
case <-c.ctx.Done():
abortWaiters(c.error())
// Abort all active calls. This will also prevent any new calls from being added
// to waiters.
waiters.abort(c.error())
return
}
}
@@ -347,7 +388,7 @@ func filterCloseErr(err error) error {
return nil
case err == io.EOF:
return ErrClosed
case errors.Cause(err) == io.EOF:
case errors.Is(err, io.EOF):
return ErrClosed
case strings.Contains(err.Error(), "use of closed network connection"):
return ErrClosed

View File

@@ -17,8 +17,9 @@
package ttrpc
import (
"fmt"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
)
type codec struct{}
@@ -28,7 +29,7 @@ func (c codec) Marshal(msg interface{}) ([]byte, error) {
case proto.Message:
return proto.Marshal(v)
default:
return nil, errors.Errorf("ttrpc: cannot marshal unknown type: %T", msg)
return nil, fmt.Errorf("ttrpc: cannot marshal unknown type: %T", msg)
}
}
@@ -37,6 +38,6 @@ func (c codec) Unmarshal(p []byte, msg interface{}) error {
case proto.Message:
return proto.Unmarshal(p, v)
default:
return errors.Errorf("ttrpc: cannot unmarshal into unknown type: %T", msg)
return fmt.Errorf("ttrpc: cannot unmarshal into unknown type: %T", msg)
}
}

View File

@@ -16,7 +16,7 @@
package ttrpc
import "github.com/pkg/errors"
import "errors"
type serverConfig struct {
handshaker Handshaker

View File

@@ -18,6 +18,7 @@ package ttrpc
import (
"context"
"errors"
"io"
"math/rand"
"net"
@@ -25,7 +26,6 @@ import (
"sync/atomic"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

View File

@@ -18,13 +18,14 @@ package ttrpc
import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
"unsafe"
"github.com/gogo/protobuf/proto"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -51,7 +52,7 @@ func newServiceSet(interceptor UnaryServerInterceptor) *serviceSet {
func (s *serviceSet) register(name string, methods map[string]Method) {
if _, ok := s.services[name]; ok {
panic(errors.Errorf("duplicate service %v registered", name))
panic(fmt.Errorf("duplicate service %v registered", name))
}
s.services[name] = ServiceDesc{
@@ -116,12 +117,12 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
func (s *serviceSet) resolve(service, method string) (Method, error) {
srv, ok := s.services[service]
if !ok {
return nil, status.Errorf(codes.NotFound, "service %v", service)
return nil, status.Errorf(codes.Unimplemented, "service %v", service)
}
mthd, ok := srv.Methods[method]
if !ok {
return nil, status.Errorf(codes.NotFound, "method %v", method)
return nil, status.Errorf(codes.Unimplemented, "method %v", method)
}
return mthd, nil

View File

@@ -18,11 +18,12 @@ package ttrpc
import (
"context"
"errors"
"fmt"
"net"
"os"
"syscall"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
@@ -31,12 +32,12 @@ type UnixCredentialsFunc func(*unix.Ucred) error
func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
uc, err := requireUnixSocket(conn)
if err != nil {
return nil, nil, errors.Wrap(err, "ttrpc.UnixCredentialsFunc: require unix socket")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: require unix socket: %w", err)
}
rs, err := uc.SyscallConn()
if err != nil {
return nil, nil, errors.Wrap(err, "ttrpc.UnixCredentialsFunc: (net.UnixConn).SyscallConn failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: (net.UnixConn).SyscallConn failed: %w", err)
}
var (
ucred *unix.Ucred
@@ -45,15 +46,15 @@ func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net
if err := rs.Control(func(fd uintptr) {
ucred, ucredErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
}); err != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: (*syscall.RawConn).Control failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: (*syscall.RawConn).Control failed: %w", err)
}
if ucredErr != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", err)
}
if err := fn(ucred); err != nil {
return nil, nil, errors.Wrapf(err, "ttrpc.UnixCredentialsFunc: credential check failed")
return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: credential check failed: %w", err)
}
return uc, ucred, nil
@@ -93,7 +94,7 @@ func requireRoot(ucred *unix.Ucred) error {
func requireUidGid(ucred *unix.Ucred, uid, gid int) error {
if (uid != -1 && uint32(uid) != ucred.Uid) || (gid != -1 && uint32(gid) != ucred.Gid) {
return errors.Wrap(syscall.EPERM, "ttrpc: invalid credentials")
return fmt.Errorf("ttrpc: invalid credentials: %v", syscall.EPERM)
}
return nil
}

View File

@@ -2,7 +2,6 @@ package units
import (
"fmt"
"regexp"
"strconv"
"strings"
)
@@ -26,16 +25,17 @@ const (
PiB = 1024 * TiB
)
type unitMap map[string]int64
type unitMap map[byte]int64
var (
decimalMap = unitMap{"k": KB, "m": MB, "g": GB, "t": TB, "p": PB}
binaryMap = unitMap{"k": KiB, "m": MiB, "g": GiB, "t": TiB, "p": PiB}
sizeRegex = regexp.MustCompile(`^(\d+(\.\d+)*) ?([kKmMgGtTpP])?[iI]?[bB]?$`)
decimalMap = unitMap{'k': KB, 'm': MB, 'g': GB, 't': TB, 'p': PB}
binaryMap = unitMap{'k': KiB, 'm': MiB, 'g': GiB, 't': TiB, 'p': PiB}
)
var decimapAbbrs = []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
var binaryAbbrs = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
var (
decimapAbbrs = []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
binaryAbbrs = []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
)
func getSizeAndUnit(size float64, base float64, _map []string) (float64, string) {
i := 0
@@ -89,20 +89,66 @@ func RAMInBytes(size string) (int64, error) {
// Parses the human-readable size string into the amount it represents.
func parseSize(sizeStr string, uMap unitMap) (int64, error) {
matches := sizeRegex.FindStringSubmatch(sizeStr)
if len(matches) != 4 {
// TODO: rewrite to use strings.Cut if there's a space
// once Go < 1.18 is deprecated.
sep := strings.LastIndexAny(sizeStr, "01234567890. ")
if sep == -1 {
// There should be at least a digit.
return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
}
var num, sfx string
if sizeStr[sep] != ' ' {
num = sizeStr[:sep+1]
sfx = sizeStr[sep+1:]
} else {
// Omit the space separator.
num = sizeStr[:sep]
sfx = sizeStr[sep+1:]
}
size, err := strconv.ParseFloat(matches[1], 64)
size, err := strconv.ParseFloat(num, 64)
if err != nil {
return -1, err
}
// Backward compatibility: reject negative sizes.
if size < 0 {
return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
}
unitPrefix := strings.ToLower(matches[3])
if mul, ok := uMap[unitPrefix]; ok {
if len(sfx) == 0 {
return int64(size), nil
}
// Process the suffix.
if len(sfx) > 3 { // Too long.
goto badSuffix
}
sfx = strings.ToLower(sfx)
// Trivial case: b suffix.
if sfx[0] == 'b' {
if len(sfx) > 1 { // no extra characters allowed after b.
goto badSuffix
}
return int64(size), nil
}
// A suffix from the map.
if mul, ok := uMap[sfx[0]]; ok {
size *= float64(mul)
} else {
goto badSuffix
}
// The suffix may have extra "b" or "ib" (e.g. KiB or MB).
switch {
case len(sfx) == 2 && sfx[1] != 'b':
goto badSuffix
case len(sfx) == 3 && sfx[1:] != "ib":
goto badSuffix
}
return int64(size), nil
badSuffix:
return -1, fmt.Errorf("invalid suffix: '%s'", sfx)
}

View File

@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package accelerators
import (

View File

@@ -25,6 +25,7 @@ import (
ptypes "github.com/gogo/protobuf/types"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials/insecure"
"github.com/google/cadvisor/container/containerd/containers"
"github.com/google/cadvisor/container/containerd/errdefs"
@@ -77,7 +78,7 @@ func Client(address, namespace string) (ContainerdClient, error) {
connParams.Backoff.BaseDelay = baseBackoffDelay
connParams.Backoff.MaxDelay = maxBackoffDelay
gopts := []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithContextDialer(dialer.ContextDialer),
grpc.WithBlock(),
grpc.WithConnectParams(connParams),

View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//This code has been taken from containerd repo to avoid large library import
// This code has been taken from containerd repo to avoid large library import
package containerd
import (

View File

@@ -37,8 +37,6 @@ import "context"
// oriented. A namespace is really just a name and a set of labels. Objects
// that belong to a namespace are returned when the namespace is assigned to a
// given context.
//
//
type Store interface {
Create(ctx context.Context, namespace string, labels map[string]string) error
Labels(ctx context.Context, namespace string) (map[string]string, error)

View File

@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package devicemapper
import (

View File

@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package devicemapper
import (

View File

@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package devicemapper
import (

View File

@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package devicemapper
import (

View File

@@ -44,6 +44,7 @@ type Node struct {
HugePages []HugePagesInfo `json:"hugepages"`
Cores []Core `json:"cores"`
Caches []Cache `json:"caches"`
Distances []uint64 `json:"distances"`
}
type Core struct {

View File

@@ -153,6 +153,10 @@ func (p testSubcontainersInfoProvider) GetMachineInfo() (*info.MachineInfo, erro
},
},
},
Distances: []uint64{
10,
12,
},
},
{
Id: 1,
@@ -260,6 +264,10 @@ func (p testSubcontainersInfoProvider) GetMachineInfo() (*info.MachineInfo, erro
Level: 3,
},
},
Distances: []uint64{
12,
10,
},
},
},
}, nil

View File

@@ -28,13 +28,14 @@ import (
var baseLabelsNames = []string{"machine_id", "system_uuid", "boot_id"}
const (
prometheusModeLabelName = "mode"
prometheusTypeLabelName = "type"
prometheusLevelLabelName = "level"
prometheusNodeLabelName = "node_id"
prometheusCoreLabelName = "core_id"
prometheusThreadLabelName = "thread_id"
prometheusPageSizeLabelName = "page_size"
prometheusModeLabelName = "mode"
prometheusTypeLabelName = "type"
prometheusLevelLabelName = "level"
prometheusNodeLabelName = "node_id"
prometheusCoreLabelName = "core_id"
prometheusThreadLabelName = "thread_id"
prometheusPageSizeLabelName = "page_size"
prometheusTargetNodeLabelName = "target_node_id"
nvmMemoryMode = "memory_mode"
nvmAppDirectMode = "app_direct_mode"
@@ -191,6 +192,15 @@ func NewPrometheusMachineCollector(i infoProvider, includedMetrics container.Met
return getHugePagesCount(machineInfo)
},
},
{
name: "machine_node_distance",
help: "Distance between NUMA node and target NUMA node.",
valueType: prometheus.GaugeValue,
extraLabels: []string{prometheusNodeLabelName, prometheusTargetNodeLabelName},
getValues: func(machineInfo *info.MachineInfo) metricValues {
return getDistance(machineInfo)
},
},
}...)
}
return c
@@ -356,3 +366,19 @@ func getCaches(machineInfo *info.MachineInfo) metricValues {
}
return mValues
}
func getDistance(machineInfo *info.MachineInfo) metricValues {
mValues := make(metricValues, 0, len(machineInfo.Topology)^2)
for _, node := range machineInfo.Topology {
nodeID := strconv.Itoa(node.Id)
for i, target := range node.Distances {
mValues = append(mValues,
metricValue{
value: float64(target),
labels: []string{nodeID, strconv.Itoa(i)},
timestamp: machineInfo.Timestamp,
})
}
}
return mValues
}

View File

@@ -38,6 +38,8 @@ const (
meminfoFile = "meminfo"
distanceFile = "distance"
sysFsCPUTopology = "topology"
// CPUPhysicalPackageID is a physical package id of cpu#. Typically corresponds to a physical socket number,
@@ -113,6 +115,10 @@ type SysFs interface {
GetCacheInfo(cpu int, cache string) (CacheInfo, error)
GetSystemUUID() (string, error)
// GetDistances returns distance array
GetDistances(string) (string, error)
// IsCPUOnline determines if CPU status from kernel hotplug machanism standpoint.
// See: https://www.kernel.org/doc/html/latest/core-api/cpu_hotplug.html
IsCPUOnline(dir string) bool
@@ -161,6 +167,15 @@ func (fs *realSysFs) GetMemInfo(nodePath string) (string, error) {
return strings.TrimSpace(string(meminfo)), err
}
func (fs *realSysFs) GetDistances(nodePath string) (string, error) {
distancePath := fmt.Sprintf("%s/%s", nodePath, distanceFile)
distance, err := ioutil.ReadFile(distancePath)
if err != nil {
return "", err
}
return strings.TrimSpace(string(distance)), err
}
func (fs *realSysFs) GetHugePagesInfo(hugePagesDirectory string) ([]os.FileInfo, error) {
return ioutil.ReadDir(hugePagesDirectory)
}

View File

@@ -243,6 +243,11 @@ func GetNodesInfo(sysFs sysfs.SysFs) ([]info.Node, int, error) {
return nil, 0, err
}
node.Distances, err = getDistances(sysFs, nodeDir)
if err != nil {
return nil, 0, err
}
nodes = append(nodes, node)
}
return nodes, allLogicalCoresCount, err
@@ -391,6 +396,27 @@ func getNodeMemInfo(sysFs sysfs.SysFs, nodeDir string) (uint64, error) {
return uint64(memory), nil
}
// getDistances returns information about distances between NUMA nodes
func getDistances(sysFs sysfs.SysFs, nodeDir string) ([]uint64, error) {
rawDistance, err := sysFs.GetDistances(nodeDir)
if err != nil {
//Ignore if per-node info is not available.
klog.Warningf("Found node without distance information, nodeDir: %s", nodeDir)
return nil, nil
}
distances := []uint64{}
for _, distance := range strings.Split(rawDistance, " ") {
distanceUint, err := strconv.ParseUint(distance, 10, 64)
if err != nil {
return nil, fmt.Errorf("cannot convert %s to int", distance)
}
distances = append(distances, distanceUint)
}
return distances, nil
}
// getCoresInfo returns information about physical cores
func getCoresInfo(sysFs sysfs.SysFs, cpuDirs []string) ([]info.Core, error) {
cores := make([]info.Core, 0, len(cpuDirs))

View File

@@ -5,13 +5,17 @@ system.
[![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk) [![Build Status](https://dev.azure.com/microsoft0235/microsoft/_apis/build/status/karrick.godirwalk?branchName=master)](https://dev.azure.com/microsoft0235/microsoft/_build/latest?definitionId=1&branchName=master)
In short, why do I use this library?
In short, why did I create this library?
1. It's faster than `filepath.Walk`.
1. It's more correct on Windows than `filepath.Walk`.
1. It's more easy to use than `filepath.Walk`.
1. It's more flexible than `filepath.Walk`.
Depending on your specific circumstances, [you might no longer need a
library for file walking in
Go](https://engineering.kablamo.com.au/posts/2021/quick-comparison-between-go-file-walk-implementations).
## Usage Example
Additional examples are provided in the `examples/` subdirectory.

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package godirwalk
@@ -22,8 +23,10 @@ type Scanner struct {
fd int // file descriptor used to read entries from directory
}
// NewScanner returns a new directory Scanner that lazily enumerates the
// contents of a single directory.
// NewScanner returns a new directory Scanner that lazily enumerates
// the contents of a single directory. To prevent resource leaks,
// caller must invoke either the Scanner's Close or Err method after
// it has completed scanning a directory.
//
// scanner, err := godirwalk.NewScanner(dirname)
// if err != nil {
@@ -52,10 +55,12 @@ func NewScanner(osDirname string) (*Scanner, error) {
return NewScannerWithScratchBuffer(osDirname, nil)
}
// NewScannerWithScratchBuffer returns a new directory Scanner that lazily
// enumerates the contents of a single directory. On platforms other than
// Windows it uses the provided scratch buffer to read from the file system. On
// Windows the scratch buffer is ignored.
// NewScannerWithScratchBuffer returns a new directory Scanner that
// lazily enumerates the contents of a single directory. On platforms
// other than Windows it uses the provided scratch buffer to read from
// the file system. On Windows the scratch buffer is ignored. To
// prevent resource leaks, caller must invoke either the Scanner's
// Close or Err method after it has completed scanning a directory.
func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) {
dh, err := os.Open(osDirname)
if err != nil {
@@ -73,6 +78,13 @@ func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scann
return scanner, nil
}
// Close releases resources associated with scanning a directory. Call
// either this or the Err method when the directory no longer needs to
// be scanned.
func (s *Scanner) Close() error {
return s.Err()
}
// Dirent returns the current directory entry while scanning a directory.
func (s *Scanner) Dirent() (*Dirent, error) {
if s.de == nil {
@@ -90,8 +102,10 @@ func (s *Scanner) done(err error) {
return
}
if cerr := s.dh.Close(); err == nil {
s.err = cerr
s.err = err
if err = s.dh.Close(); s.err == nil {
s.err = err
}
s.osDirname, s.childName = "", ""
@@ -101,9 +115,10 @@ func (s *Scanner) done(err error) {
s.fd = 0
}
// Err returns any error associated with scanning a directory. It is normal to
// call Err after Scan returns false, even though they both ensure Scanner
// resources are released. Do not call until done scanning a directory.
// Err returns any error associated with scanning a directory. It is
// normal to call Err after Scan returns false, even though they both
// ensure Scanner resources are released. Call either this or the
// Close method when the directory no longer needs to be scanned.
func (s *Scanner) Err() error {
s.done(nil)
return s.err
@@ -135,7 +150,7 @@ func (s *Scanner) Scan() bool {
if err == syscall.EINTR /* || err == unix.EINTR */ {
continue
}
s.done(err)
s.done(err) // any other error forces a stop
return false
}
if n <= 0 { // end of directory: normal exit

View File

@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package godirwalk
@@ -17,8 +18,10 @@ type Scanner struct {
childMode os.FileMode
}
// NewScanner returns a new directory Scanner that lazily enumerates the
// contents of a single directory.
// NewScanner returns a new directory Scanner that lazily enumerates
// the contents of a single directory. To prevent resource leaks,
// caller must invoke either the Scanner's Close or Err method after
// it has completed scanning a directory.
//
// scanner, err := godirwalk.NewScanner(dirname)
// if err != nil {
@@ -55,14 +58,24 @@ func NewScanner(osDirname string) (*Scanner, error) {
return scanner, nil
}
// NewScannerWithScratchBuffer returns a new directory Scanner that lazily
// enumerates the contents of a single directory. On platforms other than
// Windows it uses the provided scratch buffer to read from the file system. On
// Windows the scratch buffer parameter is ignored.
// NewScannerWithScratchBuffer returns a new directory Scanner that
// lazily enumerates the contents of a single directory. On platforms
// other than Windows it uses the provided scratch buffer to read from
// the file system. On Windows the scratch buffer parameter is
// ignored. To prevent resource leaks, caller must invoke either the
// Scanner's Close or Err method after it has completed scanning a
// directory.
func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) {
return NewScanner(osDirname)
}
// Close releases resources associated with scanning a directory. Call
// either this or the Err method when the directory no longer needs to
// be scanned.
func (s *Scanner) Close() error {
return s.Err()
}
// Dirent returns the current directory entry while scanning a directory.
func (s *Scanner) Dirent() (*Dirent, error) {
if s.de == nil {
@@ -83,17 +96,20 @@ func (s *Scanner) done(err error) {
return
}
if cerr := s.dh.Close(); err == nil {
s.err = cerr
s.err = err
if err = s.dh.Close(); s.err == nil {
s.err = err
}
s.childName, s.osDirname = "", ""
s.de, s.dh = nil, nil
}
// Err returns any error associated with scanning a directory. It is normal to
// call Err after Scan returns false, even though they both ensure Scanner
// resources are released. Do not call until done scanning a directory.
// Err returns any error associated with scanning a directory. It is
// normal to call Err after Scan returns false, even though they both
// ensure Scanner resources are released. Call either this or the
// Close method when the directory no longer needs to be scanned.
func (s *Scanner) Err() error {
s.done(nil)
return s.err

View File

@@ -15,7 +15,7 @@ import (
//
// If a non-existent path is specified, an appropriate error is returned.
// In case the caller is not interested in this particular error, it should
// be handled separately using e.g. errors.Is(err, os.ErrNotExist).
// be handled separately using e.g. errors.Is(err, fs.ErrNotExist).
//
// This function is only available on Linux. When available (since kernel
// v5.6), openat2(2) syscall is used to reliably detect all mounts. Otherwise,

View File

@@ -1,10 +1,9 @@
//go:build linux || (freebsd && cgo) || (openbsd && cgo) || (darwin && cgo)
// +build linux freebsd,cgo openbsd,cgo darwin,cgo
//go:build linux || freebsd || openbsd || darwin
// +build linux freebsd openbsd darwin
package mountinfo
import (
"fmt"
"os"
"path/filepath"
@@ -33,13 +32,13 @@ func mountedByStat(path string) (bool, error) {
func normalizePath(path string) (realPath string, err error) {
if realPath, err = filepath.Abs(path); err != nil {
return "", fmt.Errorf("unable to get absolute path for %q: %w", path, err)
return "", err
}
if realPath, err = filepath.EvalSymlinks(realPath); err != nil {
return "", fmt.Errorf("failed to canonicalise path for %q: %w", path, err)
return "", err
}
if _, err := os.Stat(realPath); err != nil {
return "", fmt.Errorf("failed to stat target of %q: %w", path, err)
return "", err
}
return realPath, nil
}

View File

@@ -15,7 +15,7 @@ func GetMounts(f FilterFunc) ([]*Info, error) {
//
// If a non-existent path is specified, an appropriate error is returned.
// In case the caller is not interested in this particular error, it should
// be handled separately using e.g. errors.Is(err, os.ErrNotExist).
// be handled separately using e.g. errors.Is(err, fs.ErrNotExist).
func Mounted(path string) (bool, error) {
// root is always mounted
if path == string(os.PathSeparator) {

View File

@@ -1,53 +1,37 @@
//go:build (freebsd && cgo) || (openbsd && cgo) || (darwin && cgo)
// +build freebsd,cgo openbsd,cgo darwin,cgo
//go:build freebsd || openbsd || darwin
// +build freebsd openbsd darwin
package mountinfo
/*
#include <sys/param.h>
#include <sys/ucred.h>
#include <sys/mount.h>
*/
import "C"
import (
"fmt"
"reflect"
"unsafe"
)
import "golang.org/x/sys/unix"
// parseMountTable returns information about mounted filesystems
func parseMountTable(filter FilterFunc) ([]*Info, error) {
var rawEntries *C.struct_statfs
count := int(C.getmntinfo(&rawEntries, C.MNT_WAIT))
if count == 0 {
return nil, fmt.Errorf("failed to call getmntinfo")
count, err := unix.Getfsstat(nil, unix.MNT_WAIT)
if err != nil {
return nil, err
}
var entries []C.struct_statfs
header := (*reflect.SliceHeader)(unsafe.Pointer(&entries))
header.Cap = count
header.Len = count
header.Data = uintptr(unsafe.Pointer(rawEntries))
entries := make([]unix.Statfs_t, count)
_, err = unix.Getfsstat(entries, unix.MNT_WAIT)
if err != nil {
return nil, err
}
var out []*Info
for _, entry := range entries {
var mountinfo Info
var skip, stop bool
mountinfo.Mountpoint = C.GoString(&entry.f_mntonname[0])
mountinfo.FSType = C.GoString(&entry.f_fstypename[0])
mountinfo.Source = C.GoString(&entry.f_mntfromname[0])
mountinfo := getMountinfo(&entry)
if filter != nil {
// filter out entries we're not interested in
skip, stop = filter(&mountinfo)
skip, stop = filter(mountinfo)
if skip {
continue
}
}
out = append(out, &mountinfo)
out = append(out, mountinfo)
if stop {
break
}

View File

@@ -0,0 +1,14 @@
//go:build freebsd || darwin
// +build freebsd darwin
package mountinfo
import "golang.org/x/sys/unix"
func getMountinfo(entry *unix.Statfs_t) *Info {
return &Info{
Mountpoint: unix.ByteSliceToString(entry.Mntonname[:]),
FSType: unix.ByteSliceToString(entry.Fstypename[:]),
Source: unix.ByteSliceToString(entry.Mntfromname[:]),
}
}

View File

@@ -0,0 +1,11 @@
package mountinfo
import "golang.org/x/sys/unix"
func getMountinfo(entry *unix.Statfs_t) *Info {
return &Info{
Mountpoint: unix.ByteSliceToString(entry.F_mntonname[:]),
FSType: unix.ByteSliceToString(entry.F_fstypename[:]),
Source: unix.ByteSliceToString(entry.F_mntfromname[:]),
}
}

View File

@@ -1,5 +1,5 @@
//go:build (!windows && !linux && !freebsd && !openbsd && !darwin) || (freebsd && !cgo) || (openbsd && !cgo) || (darwin && !cgo)
// +build !windows,!linux,!freebsd,!openbsd,!darwin freebsd,!cgo openbsd,!cgo darwin,!cgo
//go:build !windows && !linux && !freebsd && !openbsd && !darwin
// +build !windows,!linux,!freebsd,!openbsd,!darwin
package mountinfo

View File

@@ -66,10 +66,6 @@ func StdStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) {
}
}
// Temporarily use STD_INPUT_HANDLE, STD_OUTPUT_HANDLE and
// STD_ERROR_HANDLE from syscall rather than x/sys/windows as long as
// go-ansiterm hasn't switch to x/sys/windows.
// TODO: switch back to x/sys/windows once go-ansiterm has switched
if emulateStdin {
h := uint32(windows.STD_INPUT_HANDLE)
stdIn = windowsconsole.NewAnsiReader(int(h))