diff --git a/pkg/kubectl/cmd/set/set_env.go b/pkg/kubectl/cmd/set/set_env.go
index abe4932a071..3b2dd4ab561 100644
--- a/pkg/kubectl/cmd/set/set_env.go
+++ b/pkg/kubectl/cmd/set/set_env.go
@@ -231,7 +231,7 @@ func (o *EnvOptions) RunEnv(f cmdutil.Factory) error {
 	}
 
 	if len(o.From) != 0 {
-		b := f.NewBuilder(!o.Local).
+		b := f.NewBuilder().
 			ContinueOnError().
 			NamespaceParam(cmdNamespace).DefaultNamespace().
 			FilenameParam(enforceNamespace, &o.FilenameOptions).
@@ -242,6 +242,8 @@ func (o *EnvOptions) RunEnv(f cmdutil.Factory) error {
 				SelectorParam(o.Selector).
 				ResourceTypeOrNameArgs(o.All, o.From).
 				Latest()
+		} else {
+			b = b.Local(f.ClientForMapping)
 		}
 
 		infos, err := b.Do().Infos()
@@ -293,7 +295,7 @@ func (o *EnvOptions) RunEnv(f cmdutil.Factory) error {
 		}
 	}
 
-	b := f.NewBuilder(!o.Local).
+	b := f.NewBuilder().
 		ContinueOnError().
 		NamespaceParam(cmdNamespace).DefaultNamespace().
 		FilenameParam(enforceNamespace, &o.FilenameOptions).
@@ -304,6 +306,8 @@ func (o *EnvOptions) RunEnv(f cmdutil.Factory) error {
 			SelectorParam(o.Selector).
 			ResourceTypeOrNameArgs(o.All, o.Resources...).
 			Latest()
+	} else {
+		b = b.Local(f.ClientForMapping)
 	}
 
 	o.Infos, err = b.Do().Infos()
diff --git a/pkg/kubectl/cmd/testing/BUILD b/pkg/kubectl/cmd/testing/BUILD
index 6f66e1e84ab..808e9e57116 100644
--- a/pkg/kubectl/cmd/testing/BUILD
+++ b/pkg/kubectl/cmd/testing/BUILD
@@ -28,7 +28,6 @@ go_library(
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
diff --git a/pkg/kubectl/resource/BUILD b/pkg/kubectl/resource/BUILD
index 1858e130ba4..21b07ffeb93 100644
--- a/pkg/kubectl/resource/BUILD
+++ b/pkg/kubectl/resource/BUILD
@@ -29,6 +29,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",