package fs import ( "context" "os" "path/filepath" "strings" "sync" "github.com/Sirupsen/logrus" ) // ChangeKind is the type of modification that // a change is making. type ChangeKind int const ( // ChangeKindAdd represents an addition of // a file ChangeKindAdd = iota // ChangeKindModify represents a change to // an existing file ChangeKindModify // ChangeKindDelete represents a delete of // a file ChangeKindDelete ) func (k ChangeKind) String() string { switch k { case ChangeKindAdd: return "add" case ChangeKindModify: return "modify" case ChangeKindDelete: return "delete" default: return "" } } // Change represents single change between a diff and its parent. type Change struct { Kind ChangeKind Path string FileInfo os.FileInfo Source string } // Changes returns a stream of changes between the provided upper // directory and lower directory. // // Changes are ordered by name and should be appliable in the // order in which they received. // Due to this apply ordering, the following is true // - Removed directory trees only create a single change for the root // directory removed. Remaining changes are implied. // - A directory which is modified to become a file will not have // delete entries for sub-path items, their removal is implied // by the removal of the parent directory. // // Opaque directories will not be treated specially and each file // removed from the lower will show up as a removal // // File content comparisons will be done on files which have timestamps // which may have been truncated. If either of the files being compared // has a zero value nanosecond value, each byte will be compared for // differences. If 2 files have the same seconds value but different // nanosecond values where one of those values is zero, the files will // be considered unchanged if the content is the same. This behavior // is to account for timestamp truncation during archiving. func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) { var ( changes = make(chan Change) retCtx, cancel = context.WithCancel(ctx) ) cc := &changeContext{ Context: retCtx, } go func() { var err error if lower == "" { logrus.Debugf("Using single walk diff for %s", upper) err = addDirChanges(ctx, changes, upper) } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) err = diffDirChanges(ctx, changes, lower, diffOptions) } else { logrus.Debugf("Using double walk diff for %s from %s", upper, lower) err = doubleWalkDiff(ctx, changes, upper, lower) } if err != nil { cc.errL.Lock() cc.err = err cc.errL.Unlock() cancel() } defer close(changes) }() return cc, changes } // changeContext wraps a context to allow setting an error // directly from a change streamer to allow streams canceled // due to errors to propagate the error to the caller. type changeContext struct { context.Context err error errL sync.Mutex } func (cc *changeContext) Err() error { cc.errL.Lock() if cc.err != nil { return cc.err } cc.errL.Unlock() return cc.Context.Err() } func sendChange(ctx context.Context, changes chan<- Change, change Change) error { select { case <-ctx.Done(): return ctx.Err() case changes <- change: return nil } } func addDirChanges(ctx context.Context, changes chan<- Change, root string) error { return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { if err != nil { return err } // Rebase path path, err = filepath.Rel(root, path) if err != nil { return err } path = filepath.Join(string(os.PathSeparator), path) // Skip root if path == string(os.PathSeparator) { return nil } change := Change{ Path: path, Kind: ChangeKindAdd, FileInfo: f, Source: filepath.Join(root, path), } return sendChange(ctx, changes, change) }) } // diffDirOptions is used when the diff can be directly calculated from // a diff directory to its lower, without walking both trees. type diffDirOptions struct { diffDir string skipChange func(string) (bool, error) deleteChange func(string, string, os.FileInfo) (string, error) } // diffDirChanges walks the diff directory and compares changes against the lower. func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error { changedDirs := make(map[string]struct{}) return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { if err != nil { return err } // Rebase path path, err = filepath.Rel(o.diffDir, path) if err != nil { return err } path = filepath.Join(string(os.PathSeparator), path) // Skip root if path == string(os.PathSeparator) { return nil } // TODO: handle opaqueness, start new double walker at this // location to get deletes, and skip tree in single walker if o.skipChange != nil { if skip, err := o.skipChange(path); skip { return err } } change := Change{ Path: path, } deletedFile, err := o.deleteChange(o.diffDir, path, f) if err != nil { return err } // Find out what kind of modification happened if deletedFile != "" { change.Path = deletedFile change.Kind = ChangeKindDelete } else { // Otherwise, the file was added change.Kind = ChangeKindAdd change.FileInfo = f change.Source = filepath.Join(o.diffDir, path) // ...Unless it already existed in a lower, in which case, it's a modification stat, err := os.Stat(filepath.Join(lower, path)) if err != nil && !os.IsNotExist(err) { return err } if err == nil { // The file existed in the lower, so that's a modification // However, if it's a directory, maybe it wasn't actually modified. // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar if stat.IsDir() && f.IsDir() { if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { // Both directories are the same, don't record the change return nil } } change.Kind = ChangeKindModify } } // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. // This block is here to ensure the change is recorded even if the // modify time, mode and size of the parent directory in the rw and ro layers are all equal. // Check https://github.com/docker/docker/pull/13590 for details. if f.IsDir() { changedDirs[path] = struct{}{} } if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete { parent := filepath.Dir(path) if _, ok := changedDirs[parent]; !ok && parent != "/" { pi, err := os.Stat(filepath.Join(o.diffDir, parent)) if err != nil { return err } dirChange := Change{ Path: parent, Kind: ChangeKindModify, FileInfo: pi, Source: filepath.Join(o.diffDir, parent), } if err := sendChange(ctx, changes, dirChange); err != nil { return err } changedDirs[parent] = struct{}{} } } return sendChange(ctx, changes, change) }) } // doubleWalkDiff walks both directories to create a diff func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) { pathCtx, cancel := context.WithCancel(ctx) defer func() { if err != nil { cancel() } }() var ( w1 = pathWalker(pathCtx, lower) w2 = pathWalker(pathCtx, upper) f1, f2 *currentPath rmdir string ) for w1 != nil || w2 != nil { if f1 == nil && w1 != nil { f1, err = nextPath(w1) if err != nil { return err } if f1 == nil { w1 = nil } } if f2 == nil && w2 != nil { f2, err = nextPath(w2) if err != nil { return err } if f2 == nil { w2 = nil } } if f1 == nil && f2 == nil { continue } c := pathChange(f1, f2) switch c.Kind { case ChangeKindAdd: if rmdir != "" { rmdir = "" } c.FileInfo = f2.f c.Source = filepath.Join(upper, c.Path) f2 = nil case ChangeKindDelete: // Check if this file is already removed by being // under of a removed directory if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { f1 = nil continue } else if rmdir == "" && f1.f.IsDir() { rmdir = f1.path + string(os.PathSeparator) } else if rmdir != "" { rmdir = "" } f1 = nil case ChangeKindModify: same, err := sameFile(f1, f2) if err != nil { return err } if f1.f.IsDir() && !f2.f.IsDir() { rmdir = f1.path + string(os.PathSeparator) } else if rmdir != "" { rmdir = "" } c.FileInfo = f2.f c.Source = filepath.Join(upper, c.Path) f1 = nil f2 = nil if same { continue } } if err := sendChange(ctx, changes, c); err != nil { return err } } return nil }