build: Release v15.0

The Rust manifiest requires a three part version number however we continue with our plan to use two digits so the last part will not be used. Signed-off-by: Rob Bradford <robert.bradford@intel.com>
tests: Fix reboot by allowing specifying the custom timeout
2026-08-05 02:19:16 +00:00 · 2021-04-29 16:07:45 +01:00 · 2021-04-29 11:10:40 +01:00 · 2021-04-29 11:10:32 +01:00 · 2021-04-29 09:44:40 +01:00 · 2021-04-29 09:44:40 +01:00
93 changed files with 2219 additions and 2107 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: File a bug report
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behaviour:
+
+**Version**
+
+Output of `cloud-hypervisor --version`:
+
+Did you build from source, if so build command line (e.g. features):
+
+**VM configuration**
+
+What command line did you run (or JSON config data):
+
+Guest OS version details:
+
+Host OS version details:
+
+**Logs**
+
+Output of `cloud-hypervisor -v` from either standard error or via `--log-file`:
+
+Linux kernel output:
--- a/.github/workflows/quality-aarch64.yaml
+++ b/.github/workflows/quality-aarch64.yaml
@@ -6,12 +6,19 @@ jobs:
    if: github.event_name == 'pull_request'
    name: Quality (clippy, rustfmt)
    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
    strategy:
+      fail-fast: false
      matrix:
        rust:
          - stable
        target:
          - aarch64-unknown-linux-gnu
+        experimental: [false]
+        include:
+          - rust: beta
+            target: aarch64-unknown-linux-gnu
+            experimental: true
    steps:
      - name: Code checkout
        uses: actions/checkout@v2
--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -6,12 +6,19 @@ jobs:
    if: github.event_name == 'pull_request'
    name: Quality (clippy, rustfmt)
    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
    strategy:
+      fail-fast: false
      matrix:
        rust:
          - stable
        target:
          - x86_64-unknown-linux-gnu
+        experimental: [false]
+        include:
+          - rust: beta
+            target: x86_64-unknown-linux-gnu
+            experimental: true
    steps:
      - name: Code checkout
        uses: actions/checkout@v2
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -42,9 +42,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cddc5f91628367664cc7c69714ff08deee8a3efc54623011c772544d7b2767"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"

 [[package]]
 name = "api_client"
@@ -80,18 +80,6 @@ dependencies = [
 name = "arch_gen"
 version = "0.1.0"

-[[package]]
-name = "arrayref"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
-
-[[package]]
-name = "arrayvec"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -111,11 +99,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

 [[package]]
 name = "backtrace"
-version = "0.3.56"
+version = "0.3.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
+checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82"
 dependencies = [
 "addr2line",
+ "cc",
 "cfg-if 1.0.0",
 "libc",
 "miniz_oxide",
@@ -123,12 +112,6 @@ dependencies = [
 "rustc-demangle",
 ]

-[[package]]
-name = "base64"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
-
 [[package]]
 name = "bitflags"
 version = "0.5.0"
@@ -141,17 +124,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

-[[package]]
-name = "blake2b_simd"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"
-dependencies = [
- "arrayref",
- "arrayvec",
- "constant_time_eq",
-]
-
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -212,7 +184,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "0.14.0"
+version = "15.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -247,12 +219,6 @@ dependencies = [
 "bitflags 1.2.1",
 ]

-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
 [[package]]
 name = "credibility"
 version = "0.1.3"
@@ -263,17 +229,6 @@ dependencies = [
 "failure_derive",
 ]

-[[package]]
-name = "crossbeam-utils"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
-dependencies = [
- "autocfg",
- "cfg-if 1.0.0",
- "lazy_static",
-]
-
 [[package]]
 name = "devices"
 version = "0.1.0"
@@ -296,18 +251,18 @@ dependencies = [

 [[package]]
 name = "dirs"
-version = "3.0.1"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff"
+checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309"
 dependencies = [
 "dirs-sys",
 ]

 [[package]]
 name = "dirs-sys"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
+checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
 dependencies = [
 "libc",
 "redox_users",
@@ -371,9 +326,9 @@ dependencies = [

 [[package]]
 name = "getrandom"
-version = "0.1.16"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
 "cfg-if 1.0.0",
 "libc",
@@ -431,9 +386,9 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4261b15a556aa9b83b976b7aa4613f5a3b80351683af14c5d8d7e62c00869d5a"
+checksum = "97b85a147f5dcb22ece887da5b99187954cc046939f22416e953dd7c336e85a1"
 dependencies = [
 "lazy_static",
 "static_assertions",
@@ -503,9 +458,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.91"
+version = "0.2.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
+checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"

 [[package]]
 name = "libssh2-sys"
@@ -578,9 +533,10 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=master#59ab64440a95f7b16253fef67b5201c2ec4adaf7"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9b605a8b61df602cda62076d30d9f70307ea336f"
 dependencies = [
- "epoll",
+ "libc",
+ "vmm-sys-util",
 ]

 [[package]]
@@ -596,7 +552,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#1c2ae9a3bc67783f7ffe792e31de97cb851b6f4e"
+source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#936e2e34af0af50f4383ab8a208e03dfba7ed206"
 dependencies = [
 "libc",
 "serde",
@@ -608,7 +564,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#1c2ae9a3bc67783f7ffe792e31de97cb851b6f4e"
+source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#936e2e34af0af50f4383ab8a208e03dfba7ed206"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -632,6 +588,7 @@ dependencies = [
 "log 0.4.14",
 "net_gen",
 "pnet",
+ "rate_limiter",
 "serde",
 "serde_json",
 "virtio-bindings",
@@ -648,9 +605,9 @@ checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"

 [[package]]
 name = "openssl-sys"
-version = "0.9.61"
+version = "0.9.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f"
+checksum = "fa52160d45fa2e7608d504b7c3a3355afed615e6d8b627a74458634ba21b69bd"
 dependencies = [
 "autocfg",
 "cc",
@@ -682,7 +639,7 @@ dependencies = [
 "cfg-if 0.1.10",
 "cloudabi",
 "libc",
- "redox_syscall",
+ "redox_syscall 0.1.57",
 "smallvec",
 "winapi 0.3.9",
 ]
@@ -804,9 +761,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.24"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
 dependencies = [
 "unicode-xid 0.2.1",
 ]
@@ -831,6 +788,15 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "rate_limiter"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "log 0.4.14",
+ "vmm-sys-util",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.57"
@@ -838,21 +804,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"

 [[package]]
-name = "redox_users"
-version = "0.3.5"
+name = "redox_syscall"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
+checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
+dependencies = [
+ "bitflags 1.2.1",
+]
+
+[[package]]
+name = "redox_users"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
 "getrandom",
- "redox_syscall",
- "rust-argon2",
+ "redox_syscall 0.2.6",
 ]

 [[package]]
 name = "regex"
-version = "1.4.5"
+version = "1.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
+checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -876,18 +850,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "rust-argon2"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
-dependencies = [
- "base64",
- "blake2b_simd",
- "constant_time_eq",
- "crossbeam-utils",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.18"
@@ -915,7 +877,7 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 [[package]]
 name = "seccomp"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.22.0#cc5387637c132e500b4857b80b5a239d8d732b77"
+source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.2#5ba819d7b7a684107f5434bcbd1617bc94565478"
 dependencies = [
 "libc",
 ]
@@ -950,9 +912,9 @@ dependencies = [

 [[package]]
 name = "signal-hook"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6aa894ef3fade0ee7243422f4fbbd6c2b48e6de767e621d37ef65f2310f53cea"
+checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac"
 dependencies = [
 "libc",
 "signal-hook-registry",
@@ -999,9 +961,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

 [[package]]
 name = "syn"
-version = "1.0.64"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f"
+checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1160,9 +1122,9 @@ checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"

 [[package]]
 name = "vcpkg"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
+checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d"

 [[package]]
 name = "vec_map"
@@ -1196,7 +1158,7 @@ dependencies = [
 [[package]]
 name = "vhost"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vhost?branch=master#05cd8b2ad3c7bb1a84f15b629711d1886b97b7ce"
+source = "git+https://github.com/rust-vmm/vhost?branch=master#c4cd1d375e386069d0b781174de9f62ef41812e2"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -1274,6 +1236,7 @@ dependencies = [
 "net_gen",
 "net_util",
 "pci",
+ "rate_limiter",
 "seccomp",
 "serde",
 "serde_derive",
@@ -1410,9 +1373,9 @@ dependencies = [

 [[package]]
 name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
+version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"

 [[package]]
 name = "winapi"
@@ -1469,9 +1432,9 @@ dependencies = [

 [[package]]
 name = "zerocopy-derive"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
+checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495"
 dependencies = [
 "proc-macro2",
 "syn",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.14.0"
+version = "15.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
@@ -13,18 +13,18 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 lto = true

 [dependencies]
-anyhow = "1.0.39"
+anyhow = "1.0.40"
 api_client = { path = "api_client" }
 clap = { version = "2.33.3", features = ["wrap_help"] }
 epoll = "4.3.1"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.91"
+libc = "0.2.94"
 log = { version = "0.4.14", features = ["std"] }
 option_parser = { path = "option_parser" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
 serde_json = "1.0.64"
-signal-hook = "0.3.7"
+signal-hook = "0.3.8"
 thiserror = "1.0.24"
 vmm = { path = "vmm" }
 vmm-sys-util = "0.8.0"
@@ -39,7 +39,7 @@ kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branc

 [dev-dependencies]
 credibility = "0.1.3"
-dirs = "3.0.1"
+dirs = "3.0.2"
 lazy_static= "1.4.0"
 net_util = { path = "net_util" }
 serde_json = "1.0.64"
@@ -76,6 +76,7 @@ members = [
    "option_parser",
    "pci",
    "qcow",
+    "rate_limiter",
    "vhost_user_backend",
    "vhost_user_block",
    "vhost_user_net",
--- a/README.md
+++ b/README.md
@@ -26,7 +26,8 @@

 # 1. What is Cloud Hypervisor?

-Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of [KVM](https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt).
+Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of [KVM](https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt) and the MSHV hypervisors .
+
 The project focuses on exclusively running modern, cloud workloads, on top of a limited set of hardware architectures and platforms.
 Cloud workloads refers to those that are usually run by customers inside a cloud provider. For our purposes this means modern operating systems with most I/O handled by paravirtualised devices (i.e. virtio), no requirement for legacy devices, and 64-bit CPUs.

@@ -36,7 +37,7 @@ Cloud Hypervisor is implemented in [Rust](https://www.rust-lang.org/) and is bas

 ### High Level

- KVM based
+- Runs on KVM or MSHV
 - Minimal emulation
 - Low latency
 - Low memory footprint
@@ -53,7 +54,7 @@ Cloud Hypervisor supports the `x86-64` and `AArch64` architectures. There are so

 ### Guest OS

-Cloud Hypervisor supports `64-bit Linux` with support for _modern_ 64-bit Windows guests currently under development.
+Cloud Hypervisor supports `64-bit Linux` and Windows 10/Windows Server 2019.

 # 2. Getting Started

@@ -148,7 +149,7 @@ We need to get the latest `rust-hypervisor-firmware` release and also a working
 $ pushd $CLOUDH
 $ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
 $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
-$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.0/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.1/hypervisor-fw
 $ popd
 ```

@@ -171,7 +172,7 @@ Multiple arguments can be given to the `--disk` parameter.

 #### Building your kernel

-Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel or `bzImage`. In order to support virtio-fs and virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.
+Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel. In order to support virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.

 To build the kernel:

@@ -179,7 +180,7 @@ To build the kernel:

 # Clone the Cloud Hypervisor Linux branch
 $ pushd $CLOUDH
-$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b virtio-fs-virtio-iommu-virtio-mem-5.6-rc4 linux-cloud-hypervisor
+$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-5.12 linux-cloud-hypervisor
 $ pushd linux-cloud-hypervisor

 # Use the cloud-hypervisor kernel config to build your kernel
@@ -241,23 +242,40 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \

 # 3. Status

-Cloud Hypervisor is under active development. No API or feature stability is guaranteed.
+Cloud Hypervisor is under active development. The following stability guarantees are currently made:

-As of 2020-07-02, the following cloud images are supported:
+* The API (including command line options) will not be removed or changed in a
+  breaking way without a minimum of 2 releases notice. Where possible warnings
+  will be given about the use of deprecated functionality and the deprecations
+  will be documented in the release notes.
+* Point releases will be made between individual releases where there are
+  substantial bug fixes or security issues that need to be fixed.
+
+Currently the following items are **not** guaranteed across updates:
+
+* Snapshot/restore is not supported across different versions
+* Live migration is not supported across different versions
+* The following features are considered experimental and may change
+  substantially between releases: TDX, SGX.
+
+
+As of 2021-04-29, the following cloud images are supported:

 - [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
 - [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (cloudimg)
+- [Ubuntu Groovy](https://cloud-images.ubuntu.com/groovy/current/) (cloudimg)
+- [Ubuntu Hirsute](https://cloud-images.ubuntu.com/hirsute/current/) (cloudimg)

 Direct kernel boot to userspace should work with a rootfs from most distributions.

 ## Hot Plug

-Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO), `virtio-{net,block,pmem,fs,vsock}` and memory resizing. This [document](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md) details how to add devices to
+Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO), `virtio-{net,block,pmem,fs,vsock}` and memory resizing. This [document](docs/hotplug.md) details how to add devices to
 a running VM.

 ## Device Model

-Details of the device model can be found in this [documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md).
+Details of the device model can be found in this [documentation](docs/device_model.md).

 ## TODO

@@ -317,3 +335,7 @@ etc, are all equal and welcome means of contribution. See the [CONTRIBUTING](CON

 Get an [invite to our Slack channel](https://join.slack.com/t/cloud-hypervisor/shared_invite/enQtNjY3MTE3MDkwNDQ4LWQ1MTA1ZDVmODkwMWQ1MTRhYzk4ZGNlN2UwNTI3ZmFlODU0OTcwOWZjMTkwZDExYWE3YjFmNzgzY2FmNDAyMjI)
 and [join us on Slack](https://cloud-hypervisor.slack.com/).
+
+## Security issues
+
+Please use the GitHub security advisories feature for reporting issues: https://github.com/cloud-hypervisor/cloud-hypervisor/security/advisories/new
--- a/acpi_tables/src/aml.rs
+++ b/acpi_tables/src/aml.rs
@@ -732,9 +732,9 @@ impl Field {
    ) -> Self {
        Field {
            path,
+            fields,
            access_type,
            update_rule,
-            fields,
        }
    }
 }
@@ -851,8 +851,8 @@ impl<'a> Aml for If<'a> {
 }

 pub struct Equal<'a> {
-    right: &'a dyn Aml,
    left: &'a dyn Aml,
+    right: &'a dyn Aml,
 }

 impl<'a> Equal<'a> {
@@ -871,8 +871,8 @@ impl<'a> Aml for Equal<'a> {
 }

 pub struct LessThan<'a> {
-    right: &'a dyn Aml,
    left: &'a dyn Aml,
+    right: &'a dyn Aml,
 }

 impl<'a> LessThan<'a> {
@@ -1056,7 +1056,7 @@ macro_rules! binary_op {

        impl<'a> $name<'a> {
            pub fn new(target: &'a dyn Aml, a: &'a dyn Aml, b: &'a dyn Aml) -> Self {
-                $name { target, a, b }
+                $name { a, b, target }
            }
        }

--- a/api_client/src/lib.rs
+++ b/api_client/src/lib.rs
@@ -76,11 +76,8 @@ impl StatusCode {

 fn get_header<'a>(res: &'a str, header: &'a str) -> Option<&'a str> {
    let header_str = format!("{}: ", header);
-    if let Some(o) = res.find(&header_str) {
-        Some(&res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
-    } else {
-        None
-    }
+    res.find(&header_str)
+        .map(|o| &res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
 }

 fn get_status_code(res: &str) -> Result<StatusCode, Error> {
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -14,7 +14,7 @@ anyhow = "1.0"
 arch_gen = { path = "../arch_gen" }
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.91"
+libc = "0.2.94"
 linux-loader = { version = "0.3.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
--- a/arch/src/aarch64/gic/gicv3.rs
+++ b/arch/src/aarch64/gic/gicv3.rs
@@ -15,8 +15,7 @@ pub mod kvm {
    use std::sync::Arc;
    use std::{boxed::Box, result};
    use vm_migration::{
-        Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-        Transportable,
+        Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable,
    };

    /// Errors thrown while saving/restoring the GICv3.
@@ -230,42 +229,15 @@ pub mod kvm {

        fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
            let gicr_typers = self.gicr_typers.clone();
-            let snapshot = serde_json::to_vec(&self.state(&gicr_typers).unwrap())
-                .map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-            let mut gic_v3_snapshot = Snapshot::new(self.id().as_str());
-            gic_v3_snapshot.add_data_section(SnapshotDataSection {
-                id: format!("{}-section", self.id()),
-                snapshot,
-            });
-
-            Ok(gic_v3_snapshot)
+            Snapshot::new_from_state(&self.id(), &self.state(&gicr_typers).unwrap())
        }

        fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-            if let Some(gic_v3_section) = snapshot
-                .snapshot_data
-                .get(&format!("{}-section", self.id()))
-            {
-                let gic_v3_state = match serde_json::from_slice(&gic_v3_section.snapshot) {
-                    Ok(state) => state,
-                    Err(error) => {
-                        return Err(MigratableError::Restore(anyhow!(
-                            "Could not deserialize GICv3 {}",
-                            error
-                        )))
-                    }
-                };
-
-                let gicr_typers = self.gicr_typers.clone();
-                return self.set_state(&gicr_typers, &gic_v3_state).map_err(|e| {
+            let gicr_typers = self.gicr_typers.clone();
+            self.set_state(&gicr_typers, &snapshot.to_state(&self.id())?)
+                .map_err(|e| {
                    MigratableError::Restore(anyhow!("Could not restore GICv3 state {:?}", e))
-                });
-            }
-
-            Err(MigratableError::Restore(anyhow!(
-                "Could not find GICv3 snapshot section"
-            )))
+                })
        }
    }

--- a/arch/src/aarch64/gic/gicv3_its.rs
+++ b/arch/src/aarch64/gic/gicv3_its.rs
@@ -3,7 +3,6 @@

 pub mod kvm {
    use std::any::Any;
-    use std::convert::TryInto;
    use std::sync::Arc;
    use std::{boxed::Box, result};
    type Result<T> = result::Result<T, Error>;
@@ -16,9 +15,6 @@ pub mod kvm {
        /// The hypervisor agnostic device
        device: Arc<dyn hypervisor::Device>,

-        /// Vector holding values of GICR_TYPER for each vCPU
-        gicr_typers: Vec<u64>,
-
        /// GIC device properties, to be used for setting up the fdt entry
        gic_properties: [u64; 4],

@@ -74,9 +70,7 @@ pub mod kvm {
            self.vcpu_count
        }

-        fn set_gicr_typers(&mut self, gicr_typers: Vec<u64>) {
-            self.gicr_typers = gicr_typers;
-        }
+        fn set_gicr_typers(&mut self, _gicr_typers: Vec<u64>) {}

        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
            self
@@ -94,7 +88,6 @@ pub mod kvm {
        ) -> Box<dyn GicDevice> {
            Box::new(KvmGicV3Its {
                device,
-                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
                gic_properties: [
                    KvmGicV3::get_dist_addr(),
                    KvmGicV3::get_dist_size(),
--- a/arch/src/aarch64/gic/mod.rs
+++ b/arch/src/aarch64/gic/mod.rs
@@ -111,10 +111,10 @@ pub mod kvm {
            flags: u32,
        ) -> Result<()> {
            let attr = kvm_bindings::kvm_device_attr {
+                flags,
                group,
                attr,
                addr,
-                flags,
            };
            device
                .set_device_attr(&attr)
@@ -132,10 +132,10 @@ pub mod kvm {
            flags: u32,
        ) -> Result<()> {
            let mut attr = kvm_bindings::kvm_device_attr {
+                flags,
                group,
                attr,
                addr,
-                flags,
            };
            device
                .get_device_attr(&mut attr)
--- a/arch/src/aarch64/layout.rs
+++ b/arch/src/aarch64/layout.rs
@@ -19,15 +19,14 @@
 // memory)   |                                                               |
 //           |                            DRAM                               |
 //           |                                                               |
-// 2GB       +---------------------------------------------------------------+
 //           |                                                               |
-//           |                           Reserved                            |
 //           |                                                               |
-// 1G+256M   +---------------------------------------------------------------+
+// 1GB       +---------------------------------------------------------------+
 //           |                                                               |
 //           |                        PCI MMCONFIG space                     |
 //           |                                                               |
-// 1GB       +---------------------------------------------------------------+
+// 768 M     +---------------------------------------------------------------+
+//           |                                                               |
 //           |                                                               |
 //           |                           PCI MMIO space                      |
 //           |                                                               |
@@ -36,8 +35,10 @@
 //           |                        Legacy devices space                   |
 //           |                                                               |
 // 144 M     +---------------------------------------------------------------|
-//           |                                                               |
 //           |                    Reserved (now GIC is here)                 |
+// 128 M     +---------------------------------------------------------------+
+//           |                                                               |
+//           |                          UEFI space                           |
 //           |                                                               |
 // 0GB       +---------------------------------------------------------------+
 //
@@ -45,6 +46,11 @@

 use vm_memory::GuestAddress;

+/// 0x0 ~ 0x800_0000 is reserved to uefi
+pub const UEFI_START: u64 = 0x0;
+pub const MEM_UEFI_START: GuestAddress = GuestAddress(0);
+pub const UEFI_SIZE: u64 = 0x0800_0000;
+
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
 pub const MAPPED_IO_START: u64 = 0x0900_0000;

@@ -53,19 +59,23 @@ pub const LEGACY_SERIAL_MAPPED_IO_START: u64 = 0x0900_0000;
 pub const LEGACY_RTC_MAPPED_IO_START: u64 = 0x0901_0000;
 pub const LEGACY_GPIO_MAPPED_IO_START: u64 = 0x0902_0000;

+/// Space 0x0902_0000 ~ 0x903_0000 is reserved for pcie io address
+pub const MEM_PCI_IO_START: GuestAddress = GuestAddress(0x0902_0000);
+pub const MEM_PCI_IO_SIZE: u64 = 0x10000;
+
 /// Legacy space will be allocated at once whiling setting up legacy devices.
 pub const LEGACY_DEVICES_MAPPED_IO_SIZE: u64 = 0x0700_0000;

-/// Starting from 0x1000_0000 (256MiB), the 768MiB (ends at 1 GiB) is used for PCIE MMIO
+/// Starting from 0x1000_0000 (256MiB) to 0x3000_0000 (768MiB) is used for PCIE MMIO
 pub const MEM_32BIT_DEVICES_START: GuestAddress = GuestAddress(0x1000_0000);
-pub const MEM_32BIT_DEVICES_SIZE: u64 = 0x3000_0000;
+pub const MEM_32BIT_DEVICES_SIZE: u64 = 0x2000_0000;

 /// PCI MMCONFIG space (start: after the device space at 1 GiB, length: 256MiB)
-pub const PCI_MMCONFIG_START: GuestAddress = GuestAddress(0x4000_0000);
+pub const PCI_MMCONFIG_START: GuestAddress = GuestAddress(0x3000_0000);
 pub const PCI_MMCONFIG_SIZE: u64 = 256 << 20;

 /// Start of RAM on 64 bit ARM.
-pub const RAM_64BIT_START: u64 = 0x8000_0000;
+pub const RAM_64BIT_START: u64 = 0x4000_0000;

 /// Kernel command line maximum size.
 /// As per `arch/arm64/include/uapi/asm/setup.h`.
@@ -74,6 +84,10 @@ pub const CMDLINE_MAX_SIZE: usize = 2048;
 /// Maximum size of the device tree blob as specified in https://www.kernel.org/doc/Documentation/arm64/booting.txt.
 pub const FDT_MAX_SIZE: usize = 0x20_0000;

+/// Put ACPI table above dtb
+pub const ACPI_START: u64 = RAM_64BIT_START + FDT_MAX_SIZE as u64;
+pub const RSDP_POINTER: GuestAddress = GuestAddress(ACPI_START);
+
 // As per virt/kvm/arm/vgic/vgic-kvm-device.c we need
 // the number of interrupts our GIC will support to be:
 // * bigger than 32
--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -10,7 +10,7 @@ io_uring = []

 [dependencies]
 io-uring = ">=0.4.0"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 qcow = { path = "../qcow" }
 serde = ">=1.0.27"
--- a/block_util/src/lib.rs
+++ b/block_util/src/lib.rs
@@ -24,7 +24,6 @@ pub mod vhd;
 use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult, DiskFileError, DiskFileResult};
 #[cfg(feature = "io_uring")]
 use io_uring::{opcode, IoUring, Probe};
-use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::convert::TryInto;
 use std::fs::File;
@@ -370,7 +369,7 @@ impl Request {
    }
 }

-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Deserialize, Serialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockConfig {
    pub capacity: u64,
@@ -393,95 +392,15 @@ pub struct VirtioBlockConfig {
    pub write_zeroes_may_unmap: u8,
    pub unused1: [u8; 3],
 }
-
-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioBlockConfig {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let capacity = self.capacity;
-        let size_max = self.size_max;
-        let seg_max = self.seg_max;
-        let geometry = self.geometry;
-        let blk_size = self.blk_size;
-        let physical_block_exp = self.physical_block_exp;
-        let alignment_offset = self.alignment_offset;
-        let min_io_size = self.min_io_size;
-        let opt_io_size = self.opt_io_size;
-        let writeback = self.writeback;
-        let unused = self.unused;
-        let num_queues = self.num_queues;
-        let max_discard_sectors = self.max_discard_sectors;
-        let max_discard_seg = self.max_discard_seg;
-        let discard_sector_alignment = self.discard_sector_alignment;
-        let max_write_zeroes_sectors = self.max_write_zeroes_sectors;
-        let max_write_zeroes_seg = self.max_write_zeroes_seg;
-        let write_zeroes_may_unmap = self.write_zeroes_may_unmap;
-        let unused1 = self.unused1;
-
-        let mut virtio_block_config = serializer.serialize_struct("VirtioBlockConfig", 60)?;
-        virtio_block_config.serialize_field("capacity", &capacity)?;
-        virtio_block_config.serialize_field("size_max", &size_max)?;
-        virtio_block_config.serialize_field("seg_max", &seg_max)?;
-        virtio_block_config.serialize_field("geometry", &geometry)?;
-        virtio_block_config.serialize_field("blk_size", &blk_size)?;
-        virtio_block_config.serialize_field("physical_block_exp", &physical_block_exp)?;
-        virtio_block_config.serialize_field("alignment_offset", &alignment_offset)?;
-        virtio_block_config.serialize_field("min_io_size", &min_io_size)?;
-        virtio_block_config.serialize_field("opt_io_size", &opt_io_size)?;
-        virtio_block_config.serialize_field("writeback", &writeback)?;
-        virtio_block_config.serialize_field("unused", &unused)?;
-        virtio_block_config.serialize_field("num_queues", &num_queues)?;
-        virtio_block_config.serialize_field("max_discard_sectors", &max_discard_sectors)?;
-        virtio_block_config.serialize_field("max_discard_seg", &max_discard_seg)?;
-        virtio_block_config
-            .serialize_field("discard_sector_alignment", &discard_sector_alignment)?;
-        virtio_block_config
-            .serialize_field("max_write_zeroes_sectors", &max_write_zeroes_sectors)?;
-        virtio_block_config.serialize_field("max_write_zeroes_seg", &max_write_zeroes_seg)?;
-        virtio_block_config.serialize_field("write_zeroes_may_unmap", &write_zeroes_may_unmap)?;
-        virtio_block_config.serialize_field("unused1", &unused1)?;
-        virtio_block_config.end()
-    }
-}
-
 unsafe impl ByteValued for VirtioBlockConfig {}

-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockGeometry {
    pub cylinders: u16,
    pub heads: u8,
    pub sectors: u8,
 }
-
-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioBlockGeometry {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let cylinders = self.cylinders;
-        let heads = self.heads;
-        let sectors = self.sectors;
-
-        let mut virtio_block_geometry = serializer.serialize_struct("VirtioBlockGeometry", 4)?;
-        virtio_block_geometry.serialize_field("cylinders", &cylinders)?;
-        virtio_block_geometry.serialize_field("heads", &heads)?;
-        virtio_block_geometry.serialize_field("sectors", &sectors)?;
-        virtio_block_geometry.end()
-    }
-}
-
 unsafe impl ByteValued for VirtioBlockGeometry {}

 /// Check if io_uring for block device can be used on the current system, as
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -8,7 +8,7 @@ anyhow = "1.0"
 bitflags = ">=1.2.1"
 byteorder = "1.4.3"
 epoll = ">=4.0.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
--- a/devices/src/ioapic.rs
+++ b/devices/src/ioapic.rs
@@ -20,16 +20,9 @@ use vm_device::interrupt::{
 };
 use vm_device::BusDevice;
 use vm_memory::GuestAddress;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

-#[derive(Serialize, Deserialize)]
-#[serde(remote = "GuestAddress")]
-pub struct GuestAddressDef(pub u64);
-
 type Result<T> = result::Result<T, Error>;

 // I/O REDIRECTION TABLE REGISTER
@@ -146,8 +139,7 @@ pub struct IoapicState {
    reg_sel: u32,
    reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
    used_entries: [bool; NUM_IOAPIC_PINS],
-    #[serde(with = "GuestAddressDef")]
-    apic_address: GuestAddress,
+    apic_address: u64,
 }

 impl BusDevice for Ioapic {
@@ -284,7 +276,7 @@ impl Ioapic {
            reg_sel: self.reg_sel,
            reg_entries: self.reg_entries,
            used_entries: self.used_entries,
-            apic_address: self.apic_address,
+            apic_address: self.apic_address.0,
        }
    }

@@ -293,7 +285,7 @@ impl Ioapic {
        self.reg_sel = state.reg_sel;
        self.reg_entries = state.reg_entries;
        self.used_entries = state.used_entries;
-        self.apic_address = state.apic_address;
+        self.apic_address = GuestAddress(state.apic_address);
        for (irq, entry) in self.used_entries.iter().enumerate() {
            if *entry {
                self.update_entry(irq)?;
@@ -418,38 +410,13 @@ impl Snapshottable for Ioapic {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut ioapic_snapshot = Snapshot::new(self.id.as_str());
-        ioapic_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(ioapic_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(ioapic_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let ioapic_state = match serde_json::from_slice(&ioapic_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize IOAPIC {}",
-                        error
-                    )))
-                }
-            };
-
-            return self.set_state(&ioapic_state).map_err(|e| {
-                MigratableError::Restore(anyhow!("Could not restore IOAPIC state {:?}", e))
-            });
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find IOAPIC snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?).map_err(|e| {
+            MigratableError::Restore(anyhow!("Could not restore state for {}: {:?}", self.id, e))
+        })
    }
 }

--- a/devices/src/legacy/gpio_pl061.rs
+++ b/devices/src/legacy/gpio_pl061.rs
@@ -8,16 +8,12 @@
 //!

 use crate::{read_le_u32, write_le_u32};
-use anyhow::anyhow;
 use std::result;
 use std::sync::{Arc, Barrier};
 use std::{fmt, io};
 use vm_device::interrupt::InterruptSourceGroup;
 use vm_device::BusDevice;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};

 const OFS_DATA: u64 = 0x400; // Data Register
 const GPIODIR: u64 = 0x400; // Direction Register
@@ -317,38 +313,12 @@ impl Snapshottable for Gpio {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut gpio_snapshot = Snapshot::new(self.id.as_str());
-        gpio_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(gpio_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(gpio_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let gpio_state = match serde_json::from_slice(&gpio_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize GPIO {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&gpio_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find the GPIO snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/devices/src/legacy/serial.rs
+++ b/devices/src/legacy/serial.rs
@@ -5,16 +5,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.

-use anyhow::anyhow;
 use std::collections::VecDeque;
 use std::sync::{Arc, Barrier};
 use std::{io, result};
 use vm_device::interrupt::InterruptSourceGroup;
 use vm_device::BusDevice;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::errno::Result;

 const LOOP_SIZE: usize = 0x40;
@@ -84,7 +80,7 @@ pub struct SerialState {
    modem_status: u8,
    scratch: u8,
    baud_divisor: u16,
-    in_buffer: VecDeque<u8>,
+    in_buffer: Vec<u8>,
 }

 impl Serial {
@@ -227,7 +223,7 @@ impl Serial {
            modem_status: self.modem_status,
            scratch: self.scratch,
            baud_divisor: self.baud_divisor,
-            in_buffer: self.in_buffer.clone(),
+            in_buffer: self.in_buffer.clone().into(),
        }
    }

@@ -240,7 +236,7 @@ impl Serial {
        self.modem_status = state.modem_status;
        self.scratch = state.scratch;
        self.baud_divisor = state.baud_divisor;
-        self.in_buffer = state.in_buffer.clone();
+        self.in_buffer = state.in_buffer.clone().into();
    }
 }

@@ -292,38 +288,12 @@ impl Snapshottable for Serial {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut serial_snapshot = Snapshot::new(self.id.as_str());
-        serial_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(serial_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(serial_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let serial_state = match serde_json::from_slice(&serial_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize SERIAL {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&serial_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find the serial snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -7,17 +7,13 @@
 //!

 use crate::{read_le_u32, write_le_u32};
-use anyhow::anyhow;
 use std::collections::VecDeque;
 use std::fmt;
 use std::sync::{Arc, Barrier};
 use std::{io, result};
 use vm_device::interrupt::InterruptSourceGroup;
 use vm_device::BusDevice;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};

 /* Registers */
 const UARTDR: u64 = 0;
@@ -99,7 +95,7 @@ pub struct Pl011State {
    dmacr: u32,
    int_enabled: u32,
    int_level: u32,
-    read_fifo: VecDeque<u8>,
+    read_fifo: Vec<u8>,
    ilpr: u32,
    ibrd: u32,
    fbrd: u32,
@@ -145,7 +141,7 @@ impl Pl011 {
            dmacr: self.dmacr,
            int_enabled: self.int_enabled,
            int_level: self.int_level,
-            read_fifo: self.read_fifo.clone(),
+            read_fifo: self.read_fifo.clone().into(),
            ilpr: self.ilpr,
            ibrd: self.ibrd,
            fbrd: self.fbrd,
@@ -163,7 +159,7 @@ impl Pl011 {
        self.dmacr = state.dmacr;
        self.int_enabled = state.int_enabled;
        self.int_level = state.int_level;
-        self.read_fifo = state.read_fifo.clone();
+        self.read_fifo = state.read_fifo.clone().into();
        self.ilpr = state.ilpr;
        self.ibrd = state.ibrd;
        self.fbrd = state.fbrd;
@@ -361,38 +357,12 @@ impl Snapshottable for Pl011 {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut pl011_snapshot = Snapshot::new(self.id.as_str());
-        pl011_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(pl011_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(pl011_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let pl011_state = match serde_json::from_slice(&pl011_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize PL011 {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&pl011_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find the PL011 snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/docs/device_model.md
+++ b/docs/device_model.md
@@ -113,9 +113,8 @@ selecting `--serial tty --console off` from the command line.
 ### virtio-iommu

 As we want to improve our nested guests support, we added support for exposing
-a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
-device through virtio. This allows for a safer nested virtio and directly
-assigned devices support.
+a [paravirtualized IOMMU](iommu.md) device through virtio. This allows for a
+safer nested virtio and directly assigned devices support.

 This device is always built-in, and it is enabled based on the presence of the
 parameter `iommu=on` in any of the virtio or VFIO devices. If at least one of
@@ -184,8 +183,8 @@ This device is always built-in, and it is enabled when `vhost_user=true` and
 shared file system, allowing for an efficient and reliable way of sharing
 a filesystem between the host and the cloud-hypervisor guest.

-See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
-documentation for more details on how to use virtio-fs with cloud-hypervisor.
+See our [filesystem sharing](fs.md) documentation for more details on how to
+use virtio-fs with cloud-hypervisor.

 This device is always built-in, and it is enabled based on the presence of the
 flag `--fs`.
@@ -206,9 +205,8 @@ VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
 access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
 physical devices into its guest.

-See our [VFIO documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
-for more details on how to directly assign host devices to `cloud-hypervisor`
-guests.
+See our [VFIO documentation](vfio.md) for more details on how to directly
+assign host devices to `cloud-hypervisor` guests.

 Because VFIO implies `vfio-pci` in the `cloud-hypervisor` context, the VFIO
 support is built-in when the `pci` feature is selected. And because the `pci`
--- a/docs/fs.md
+++ b/docs/fs.md
@@ -12,7 +12,7 @@ This virtual device relies on the _vhost-user_ protocol, which assumes the backe

 _Build virtiofsd_
 ```bash
-git clone --depth 1 "https://github.com/sboeuf/qemu.git" -b "virtio-fs" $VIRTIOFSD_DIR
+git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "qemu5.0-virtiofs-dax" $VIRTIOFSD_DIR
 cd $VIRTIOFSD_DIR
 ./configure --prefix=$PWD --target-list=x86_64-softmmu
 make virtiofsd -j `nproc`
@@ -35,45 +35,42 @@ The `cache=none` option should be the default when using `virtiofsd` with the __

 The `cache=always` option will allow for the guest page cache to be used, which will increase the memory footprint of the guest. This option should be used only for specific use cases where a single VM is going to be running on a host.

-### The kernel
+### Kernel support

-In order to leverage __virtio-fs__ support from within the guest, and because the code has not been merged in upstream Linux kernel yet, it is required to build a custom kernel embedding the patches.
-
-The following branch `virtio-fs-virtio-iommu` on the repository https://github.com/cloud-hypervisor/linux.git includes all the needed patches to support __virtio-fs__.
-
-Make sure to build a kernel out of this branch that can be then used to boot the VM.
+Modern Linux kernels starting (at least v5.10) have support for virtio-fs. Use
+of older kernels, with additional patches, are not supported.

 ## How to share directories with cloud-hypervisor

 ### Start the VM
 Once the daemon is running, the option `--fs` from __cloud-hypervisor__ needs to be used.

-Direct kernel boot option is preferred since we need to provide the custom kernel including the __virtio-fs__ patches. We could boot from `hypervisor-fw` if we had previously edited the image to replace the kernel binary.
+Direct kernel boot is the preferred option, but we can boot from an EFI cloud image if it contains a recent enough kernel.

-Because _vhost-user_ expects a dedicated process (__virtiofsd__ in this case) to be able to access the guest RAM to communicate through the _virtqueues_ with the driver running in the guest, `--memory` option needs to be slightly modified. It needs to specify a backing file for the memory so that an external process can access it.
+Because _vhost-user_ expects a dedicated process (__virtiofsd__ in this case) to be able to access the guest RAM to communicate through the _virtqueues_ with the driver running in the guest, `--memory` option needs to be slightly modified. It must specify `shared=on` to share the memory pages so that an external process can access them.

-Assuming you have `focal-server-cloudimg-amd64.raw` and `custom-vmlinux.bin` on your system, here is the __cloud-hypervisor__ command you need to run:
+Assuming you have `focal-server-cloudimg-amd64.raw` and `vmlinux` on your system, here is the __cloud-hypervisor__ command you need to run:
 ```bash
 ./cloud-hypervisor \
-    --cpus 4 \
-    --memory "size=512M,shared=on" \
+    --cpus boot=1 \
+    --memory size=1G,shared=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-vmlinux.bin \
-    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
+    --kernel vmlinux \
+    --cmdline "console=hvc0 root=/dev/vda1 rw" \
    --fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512
 ```

 By default, DAX is enabled with a cache window of 8GiB. You can specify a custom size (let's say 4GiB for this example) for the cache by explicitly setting DAX and the cache size:

 ```bash
--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G
+--fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G

 ```

 In case you don't want to use a shared window of cache to pass the shared files content, this means you will have to explicitly disable DAX with `dax=off`. Note that in this case, the `cache_size` parameter will be ignored.

 ```bash
--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off
+--fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off

 ```

--- a/docs/intel_sgx.md
+++ b/docs/intel_sgx.md
@@ -32,7 +32,7 @@ memory, the second one being 32MiB with no pre-allocated memory.
    --cpus boot=1 \
    --memory size=1G \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel bzImage \
+    --kernel vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
    --sgx-epc size=64M,prefault=on size=32M,prefault=off
 ```
--- a/docs/io_throttling.md
+++ b/docs/io_throttling.md
@@ -1,13 +1,16 @@
 # I/O Throttling

-Cloud Hypervisor now supports I/O throttling on virtio-block
+Cloud Hypervisor now supports I/O throttling on virtio-block and virtio-net
 devices. This support is based on the [`rate-limiter` module](https://github.com/firecracker-microvm/firecracker/tree/master/src/rate_limiter)
 from Firecracker. This document explains the user interface of this
 feature, and highlights some internal implementations that can help users
 better understand the expected behavior of I/O throttling in practice.

 Cloud Hypervisor allows to limit both the I/O bandwidth (e.g. bytes/s)
-and I/O operations (ops/s) independently. To limit the I/O bandwidth, it
+and I/O operations (ops/s) independently. For virtio-net devices, while
+sharing the same "rate limit" from user inputs (on both bandwidth and
+operations), the RX and TX queues are throttled independently.
+To limit the I/O bandwidth, Cloud Hypervisor
 provides three user options, i.e., `bw_size` (bytes), `bw_one_time_burst`
 (bytes), and `bw_refill_time` (ms). Both `bw_size` and `bw_refill_time`
 are required, while `bw_one_time_burst` is optional.
@@ -29,8 +32,8 @@ empty, it will stop I/O operations for a fixed amount of time
 (`cool_down_time`). The `cool_down_time` now is fixed at `100 ms`, it
 can have big implications to the actual rate limit (which can be a lot
 different the expected "refill-rate" derived from user inputs). For
-example, to have a 1000 IOPS limit, users should be able to provide
-either of the following two options:
+example, to have a 1000 IOPS limit on a virtio-blk device, users should
+be able to provide either of the following two options:
 `ops_size=1000,ops_refill_time=1000` or
 `ops_size=10,ops_refill_time=10`. However, the actual IOPS limits are
 likely to be ~1000 IOPS and ~100 IOPS respectively. The reason is the
--- a/docs/iommu.md
+++ b/docs/iommu.md
@@ -89,7 +89,7 @@ virtual IOMMU:
    --cpus boot=1 \
    --memory size=512M \
    --disk path=focal-server-cloudimg-amd64.raw,iommu=on \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
 ```

@@ -166,7 +166,7 @@ be consumed.
    --cpus boot=1 \
    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw hugepagesz=2M hugepages=2048" \
    --net tap=,mac=,iommu=on
 ```
@@ -183,7 +183,7 @@ passing through is `0000:00:01.0`.
    --cpus boot=1 \
    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts rw hugepagesz=2M hugepages=2048" \
    --device path=/sys/bus/pci/devices/0000:00:01.0,iommu=on
 ```
@@ -194,6 +194,7 @@ guest, and bind it to VFIO (it should appear as `0000:00:04.0`).
 ```bash
 echo 0000:00:04.0 > /sys/bus/pci/devices/0000\:00\:04.0/driver/unbind
 echo 8086 1502 > /sys/bus/pci/drivers/vfio-pci/new_id
+echo 0000:00:04.0 > /sys/bus/pci/drivers/vfio-pci/bind
 ```

 Last thing is to start the L2 guest with the huge pages memory backend.
@@ -203,7 +204,7 @@ Last thing is to start the L2 guest with the huge pages memory backend.
    --cpus boot=1 \
    --memory size=4G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
    --device path=/sys/bus/pci/devices/0000:00:04.0
 ```
--- a/docs/networking.md
+++ b/docs/networking.md
@@ -2,7 +2,7 @@

 cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypervisor.

-## Multiple queue support for net devices ##
+## Multiple queue support for net devices

 While multiple vcpus defined for guest, to gain the benefit of vcpu scalable to improve performance, it suggests to define multiple queue pairs for net devices, one Tx/Rx queue pair per one vcpu, that means the number of queue pairs at least is equal to the vcpu count. In that case, after virtnet driver set cpu affinity for virtqueues in guest kernel, vcpus could handle interrupt from different virtqueue pairs in parallel.

@@ -23,7 +23,7 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula

 ```bash
 ./cloud-hypervisor \
-    --cpus 4 \
+    --cpus boot=4 \
    --memory "size=512M" \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel my-vmlinux.bin \
@@ -34,14 +34,14 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula

 The `--net` argument takes 1 or more space-separated strings of key value pairs containing the following 4 keys or fields:

-| Name       | Purpose                    | Optional  |
-| -----------|----------------------------| ----------|
-| tap        | tap device name            | Yes       |
-| mac        | vNIC mac address           | Yes       |
-| ip         | tap IP IP address          | yes       |
-| mask       | tap IP netmask             | Yes       |
-| num_queues | the number of queues       | yes       |
-| queue_size | the size of each queue     | Yes       |
+| Name       | Purpose                | Optional |
+| ---------- | ---------------------- | -------- |
+| tap        | tap device name        | Yes      |
+| mac        | vNIC mac address       | Yes      |
+| ip         | tap IP IP address      | yes      |
+| mask       | tap IP netmask         | Yes      |
+| num_queues | the number of queues   | yes      |
+| queue_size | the size of each queue | Yes      |

 num_queues is the total number of tx and rx queues, the default value is 2, and it could be increased by multiples of 2. Additionally, num_queues is suggested to be as 2 times of vcpu count. The default value for queue_size is 256.

@@ -104,6 +104,7 @@ bridge name     bridge id               STP enabled     interfaces
 ich-dpl         8000.067afc1b9a67       no              ich1
 ich-int         8000.725412ffce6f       no              ich0
 ```
+
 This completes the layer 2 wiring: The cloud-hypervisor is now connected to the hypervisor host via the 2 linux bridges.

 ## IP (Layer 3) provisioning
@@ -116,6 +117,7 @@ On the hypervisor host add the network gateway IP address of each network to the
 root@host:~# ip addr add 192.168.4.1/24 dev ich-int
 root@host:~# ip addr add 10.0.1.1/24 dev ich-dpl
 ```
+
 The routing table of the hypervisor host should now also have corresponding routing entries:

 ```bash
@@ -127,9 +129,10 @@ Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
 192.168.4.0     0.0.0.0         255.255.255.0   U     0      0        0 ich-int
 192.168.178.0   0.0.0.0         255.255.255.0   U     600    0        0 wlan1
 ```
+
 ### Virtual Machine

-Within the virtual machine set the vNIC's to up state and provision the corresponding IP addresses on the 2 vNIC's. The steps outlined below use the ip command as an example. Alternative distribution specific procedures can also apply.   
+Within the virtual machine set the vNIC's to up state and provision the corresponding IP addresses on the 2 vNIC's. The steps outlined below use the ip command as an example. Alternative distribution specific procedures can also apply.

 ```bash
 root@guest:~# ip link set up enp0s2
@@ -165,7 +168,7 @@ root@192.168.4.2's password:
 Linux cloud-hypervisor 5.2.0 #2 SMP Thu Jul 11 08:08:16 CEST 2019 x86_64

 Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
-permitted by applicable law.                                                         
+permitted by applicable law.


 Last login: Fri Jul 12 13:27:56 2019 from 192.168.4.1
@@ -189,7 +192,7 @@ nameserver 192.168.178.1
 make sure that the default gateway of the hypervisor host (in this example host 192.168.178.1 which is an adsl router) has an entry in the routing table for the 192.168.4.0/24 network otherwise IP connectivity will not work.

 ```bash
-root@guest:~# nslookup  ftp.nl.debian.org       
+root@guest:~# nslookup  ftp.nl.debian.org
 Server:         192.168.178.1
 Address:        192.168.178.1#53

--- a/docs/snapshot_restore.md
+++ b/docs/snapshot_restore.md
@@ -20,7 +20,7 @@ First thing, we must run a Cloud-Hypervisor VM:
    --api-socket /tmp/cloud-hypervisor.sock \
    --cpus boot=4 \
    --memory size=4G \
-    --kernel bzImage \
+    --kernel vmlinux \
    --cmdline "root=/dev/vda1 console=hvc0 rw" \
    --disk path=focal-server-cloudimg-amd64.raw
 ```
--- a/docs/vfio.md
+++ b/docs/vfio.md
@@ -54,6 +54,7 @@ $ lspci -n -s 01:00.0
 01:00.0 ff00: 10ec:525a (rev 01)

 $ echo 10ec 525a > /sys/bus/pci/drivers/vfio-pci/new_id
+$ echo 0000:01:00.0 > /sys/bus/pci/drivers/vfio-pci/bind
 ```

 Now the device is managed by the VFIO framework.
--- a/docs/windows.md
+++ b/docs/windows.md
@@ -184,6 +184,14 @@ Set-Service -Name sshd -StartupType ‘Automatic’

 This allows for SSH login from a remote machine, for example through the `administrator` user: `ssh administrator@192.168.249.2`. For a more detailed OpenSSH guide, please follow the MSDN article from the [links](#links) section.

+## Hotplug capability
+
+CPU hotplug is supported. The VM operating system needs to support hotplug and be appropriately licensed. SKU limitations like constraints on the number of cores are to be taken into consideration. Note, that Windows doesn't support CPU hot-remove. When `ch-remote` is invoked to reduce the number of CPUs, the result will be visible after the OS reboot within the same hypervisor instance.
+
+RAM hotplug is supported. Note, that while the `pnpmem.sys` driver in use supports RAM hot-remove, the RAM being unplugged has to be not in use and have no reserved pages. In most cases it means, hot-remove won't work. Same as with the CPU hot-remove, when `ch-remote` is invoked to reduce the RAM size, the result will be visible after the OS reboot.
+
+Network device hotplug and hot-remove are supported.
+
 ## Debugging

 The Windows guest debugging process relies heavily on QEMU and [socat](http://www.dest-unreach.org/socat/). The procedure requires two Windows VMs:
--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -5,7 +5,7 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"

 [dependencies]
-libc = "0.2.91"
+libc = "0.2.94"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -20,9 +20,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cddc5f91628367664cc7c69714ff08deee8a3efc54623011c772544d7b2767"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"

 [[package]]
 name = "api_client"
@@ -135,7 +135,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -241,9 +241,9 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4261b15a556aa9b83b976b7aa4613f5a3b80351683af14c5d8d7e62c00869d5a"
+checksum = "97b85a147f5dcb22ece887da5b99187954cc046939f22416e953dd7c336e85a1"
 dependencies = [
 "lazy_static",
 "static_assertions",
@@ -294,9 +294,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.91"
+version = "0.2.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
+checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"

 [[package]]
 name = "libfuzzer-sys"
@@ -329,9 +329,10 @@ dependencies = [
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=master#59ab64440a95f7b16253fef67b5201c2ec4adaf7"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9b605a8b61df602cda62076d30d9f70307ea336f"
 dependencies = [
- "epoll",
+ "libc",
+ "vmm-sys-util",
 ]

 [[package]]
@@ -349,6 +350,7 @@ dependencies = [
 "libc",
 "log",
 "net_gen",
+ "rate_limiter",
 "serde",
 "virtio-bindings",
 "vm-memory",
@@ -383,9 +385,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.24"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
 dependencies = [
 "unicode-xid",
 ]
@@ -410,6 +412,15 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "rate_limiter"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "log",
+ "vmm-sys-util",
+]
+
 [[package]]
 name = "remain"
 version = "0.2.2"
@@ -430,7 +441,7 @@ checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 [[package]]
 name = "seccomp"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.22.0#cc5387637c132e500b4857b80b5a239d8d732b77"
+source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.2#5ba819d7b7a684107f5434bcbd1617bc94565478"
 dependencies = [
 "libc",
 ]
@@ -465,9 +476,9 @@ dependencies = [

 [[package]]
 name = "signal-hook"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6aa894ef3fade0ee7243422f4fbbd6c2b48e6de767e621d37ef65f2310f53cea"
+checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac"
 dependencies = [
 "libc",
 "signal-hook-registry",
@@ -496,9 +507,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

 [[package]]
 name = "syn"
-version = "1.0.64"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f"
+checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -618,6 +629,7 @@ dependencies = [
 "net_gen",
 "net_util",
 "pci",
+ "rate_limiter",
 "seccomp",
 "serde",
 "serde_derive",
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -10,10 +10,10 @@ cargo-fuzz = true

 [dependencies]
 block_util = { path = "../block_util" }
-libc = "0.2.90"
+libc = "0.2.94"
 libfuzzer-sys = "0.4"
 qcow = { path = "../qcow" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
 virtio-devices = { path = "../virtio-devices" }
 vmm-sys-util = "0.8.0"
 vm-virtio = { path = "../vm-virtio" }
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -14,7 +14,7 @@ tdx = []
 anyhow = "1.0"
 epoll = ">=4.0.1"
 thiserror = "1.0"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 kvm-ioctls = {  version = "0.8.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"], optional  = true }
--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -5,9 +5,10 @@ authors = ["The Chromium OS Authors"]

 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 net_gen = { path = "../net_gen" }
+rate_limiter = { path = "../rate_limiter" }
 serde = "1.0.125"
 virtio-bindings = "0.1.0"
 vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic"] }
--- a/net_util/src/lib.rs
+++ b/net_util/src/lib.rs
@@ -14,6 +14,7 @@ extern crate libc;
 #[macro_use]
 extern crate log;
 extern crate net_gen;
+extern crate rate_limiter;
 extern crate serde;
 extern crate virtio_bindings;
 extern crate vm_memory;
--- a/net_util/src/open_tap.rs
+++ b/net_util/src/open_tap.rs
@@ -70,7 +70,6 @@ pub fn open_tap(
    let mut taps: Vec<Tap> = Vec::new();
    let mut ifname: String = String::new();
    let vnet_hdr_size = vnet_hdr_len() as i32;
-    let flag = net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;

    // In case the tap interface already exists, check if the number of
    // queues is appropriate. The tap might not support multiqueue while
@@ -98,7 +97,6 @@ pub fn open_tap(
                *host_mac = Some(tap.get_mac_addr().map_err(Error::TapGetMac)?)
            }
            tap.enable().map_err(Error::TapEnable)?;
-            tap.set_offload(flag).map_err(Error::TapSetOffload)?;

            tap.set_vnet_hdr_size(vnet_hdr_size)
                .map_err(Error::TapSetVnetHdrSize)?;
@@ -106,7 +104,6 @@ pub fn open_tap(
            ifname = String::from_utf8(tap.get_if_name()).unwrap();
        } else {
            tap = Tap::open_named(ifname.as_str(), num_rx_q, flags).map_err(Error::TapOpen)?;
-            tap.set_offload(flag).map_err(Error::TapSetOffload)?;

            tap.set_vnet_hdr_size(vnet_hdr_size)
                .map_err(Error::TapSetVnetHdrSize)?;
--- a/net_util/src/queue_pair.rs
+++ b/net_util/src/queue_pair.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause

 use super::{unregister_listener, vnet_hdr_len, Tap};
+use rate_limiter::{RateLimiter, TokenType};
 use std::io;
 use std::num::Wrapping;
 use std::os::unix::io::{AsRawFd, RawFd};
@@ -36,11 +37,35 @@ impl TxVirtio {
        mem: &GuestMemoryMmap,
        tap: &mut Tap,
        queue: &mut Queue,
+        rate_limiter: &mut Option<RateLimiter>,
    ) -> Result<(), NetQueuePairError> {
        while let Some(avail_desc) = queue.iter(&mem).next() {
            let head_index = avail_desc.index;
            let mut next_desc = Some(avail_desc);

+            if let Some(rate_limiter) = rate_limiter {
+                if !rate_limiter.consume(1, TokenType::Ops) {
+                    queue.go_to_previous_position();
+                    break;
+                }
+
+                let mut bytes = Wrapping(0);
+                let mut tmp_next_desc = next_desc.clone();
+                while let Some(desc) = tmp_next_desc {
+                    if !desc.is_write_only() {
+                        bytes += Wrapping(desc.len as u64);
+                    }
+                    tmp_next_desc = desc.next_descriptor();
+                }
+                bytes -= Wrapping(vnet_hdr_len() as u64);
+                if !rate_limiter.consume(bytes.0, TokenType::Bytes) {
+                    // Revert the OPS consume().
+                    rate_limiter.manual_replenish(1, TokenType::Ops);
+                    queue.go_to_previous_position();
+                    break;
+                }
+            }
+
            let mut iovecs = Vec::new();
            while let Some(desc) = next_desc {
                if !desc.is_write_only() && desc.len > 0 {
@@ -67,8 +92,14 @@ impl TxVirtio {
                };
                if result < 0 {
                    let e = std::io::Error::last_os_error();
-                    error!("net: tx: failed writing to tap: {}", e);
                    queue.go_to_previous_position();
+
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
+                        warn!("net: tx: (recoverable) failed writing to tap: {}", e);
+                        break;
+                    }
+                    error!("net: tx: failed writing to tap: {}", e);
                    return Err(NetQueuePairError::WriteTap(e));
                }

@@ -109,9 +140,18 @@ impl RxVirtio {
        mem: &GuestMemoryMmap,
        tap: &mut Tap,
        queue: &mut Queue,
+        rate_limiter: &mut Option<RateLimiter>,
    ) -> Result<bool, NetQueuePairError> {
        let mut exhausted_descs = true;
+        let mut rate_limit_reached = false;
+
        while let Some(avail_desc) = queue.iter(&mem).next() {
+            if rate_limit_reached {
+                exhausted_descs = false;
+                queue.go_to_previous_position();
+                break;
+            }
+
            let head_index = avail_desc.index;
            let num_buffers_addr = mem.checked_offset(avail_desc.addr, 10).unwrap();
            let mut next_desc = Some(avail_desc);
@@ -145,10 +185,9 @@ impl RxVirtio {
                    exhausted_descs = false;
                    queue.go_to_previous_position();

-                    if let Some(raw_err) = e.raw_os_error() {
-                        if raw_err == libc::EAGAIN {
-                            break;
-                        }
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
+                        break;
                    }

                    error!("net: rx: failed reading from tap: {}", e);
@@ -170,6 +209,15 @@ impl RxVirtio {

            queue.add_used(&mem, head_index, len);
            queue.update_avail_event(&mem);
+
+            // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and
+            // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor
+            // chain go-through even if it was over the rate limit, and simply stop
+            // processing oncoming `avail_desc` if any.
+            if let Some(rate_limiter) = rate_limiter {
+                rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
+                    || !rate_limiter.consume(len as u64, TokenType::Bytes);
+            }
        }

        Ok(exhausted_descs)
@@ -209,6 +257,9 @@ pub struct NetQueuePair {
    pub rx_tap_listening: bool,
    pub counters: NetCounters,
    pub tap_event_id: u16,
+    pub rx_desc_avail: bool,
+    pub rx_rate_limiter: Option<RateLimiter>,
+    pub tx_rate_limiter: Option<RateLimiter>,
 }

 impl NetQueuePair {
@@ -220,7 +271,7 @@ impl NetQueuePair {
            .map(|m| m.memory())?;

        self.tx
-            .process_desc_chain(&mem, &mut self.tap, &mut queue)?;
+            .process_desc_chain(&mem, &mut self.tap, &mut queue, &mut self.tx_rate_limiter)?;

        self.counters
            .tx_bytes
@@ -241,11 +292,21 @@ impl NetQueuePair {
            .ok_or(NetQueuePairError::NoMemoryConfigured)
            .map(|m| m.memory())?;

-        if self
-            .rx
-            .process_desc_chain(&mem, &mut self.tap, &mut queue)?
-            && self.rx_tap_listening
-        {
+        self.rx_desc_avail = !self.rx.process_desc_chain(
+            &mem,
+            &mut self.tap,
+            &mut queue,
+            &mut self.rx_rate_limiter,
+        )?;
+        let rate_limit_reached = self
+            .rx_rate_limiter
+            .as_ref()
+            .map_or(false, |r| r.is_blocked());
+
+        // Stop listening on the `RX_TAP_EVENT` when:
+        // 1) there is no available describles, or
+        // 2) the RX rate limit is reached.
+        if self.rx_tap_listening && (!self.rx_desc_avail || rate_limit_reached) {
            unregister_listener(
                self.epoll_fd.unwrap(),
                self.tap.as_raw_fd(),
--- a/net_util/src/tap.rs
+++ b/net_util/src/tap.rs
@@ -190,11 +190,8 @@ impl Tap {
            return Err(Error::ConfigureTap(IoError::last_os_error()));
        }

-        let tap = Tap { if_name, tap_file };
-        let offload_flags =
-            net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;
+        let tap = Tap { tap_file, if_name };
        let vnet_hdr_size = vnet_hdr_len() as i32;
-        tap.set_offload(offload_flags)?;
        tap.set_vnet_hdr_size(vnet_hdr_size)?;

        Ok(tap)
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -10,7 +10,7 @@ byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "master" }
 vmm-sys-util = ">=0.3.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -4,11 +4,10 @@

 use crate::device::BarReprogrammingParams;
 use crate::{MsixConfig, PciInterruptPin};
-use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::fmt::{self, Display};
 use std::sync::{Arc, Mutex};
-use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable};

 // The number of 32bit registers in the config space, 4096 bytes.
 const NUM_CONFIGURATION_REGISTERS: usize = 1024;
@@ -769,85 +768,77 @@ impl PciConfiguration {

        let mask = self.writable_bits[reg_idx];
        if (BAR0_REG..BAR0_REG + NUM_BAR_REGS).contains(&reg_idx) {
+            // Ignore the case where the BAR size is being asked for.
+            if value == 0xffff_ffff {
+                return None;
+            }
+
            let bar_idx = reg_idx - 4;
-            if (value & mask) != (self.bars[bar_idx].addr & mask) {
-                // Handle special case where the address being written is
-                // different from the address initially provided. This is a
-                // BAR reprogramming case which needs to be properly caught.
-                if let Some(bar_type) = self.bars[bar_idx].r#type {
-                    match bar_type {
-                        PciBarRegionType::Memory64BitRegion => {}
-                        _ => {
-                            // Ignore the case where the BAR size is being
-                            // asked for.
-                            if value == 0xffff_ffff {
-                                return None;
-                            }
-
-                            debug!(
-                                "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                                self.registers[reg_idx], value
-                            );
-                            let old_base = u64::from(self.bars[bar_idx].addr & mask);
-                            let new_base = u64::from(value & mask);
-                            let len = u64::from(
-                                decode_32_bits_bar_size(self.bars[bar_idx].size)
-                                    .ok_or(Error::Decode32BarSize)
-                                    .unwrap(),
-                            );
-                            let region_type = bar_type;
-
-                            self.bars[bar_idx].addr = value;
-
-                            return Some(BarReprogrammingParams {
-                                old_base,
-                                new_base,
-                                len,
-                                region_type,
-                            });
-                        }
-                    }
-                } else if (reg_idx > BAR0_REG)
-                    && (self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
-                        != (self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1])
-                {
-                    // Ignore the case where the BAR size is being asked for.
-                    // Because we are in the 64bits case here, we have to check
-                    // if the lower 32bits of the current BAR have already been
-                    // asked for the BAR size too.
-                    if value == 0xffff_ffff
-                        && self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]
-                            == self.bars[bar_idx - 1].size & self.writable_bits[reg_idx - 1]
-                    {
-                        return None;
-                    }
-
-                    debug!(
-                        "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                        self.registers[reg_idx], value
-                    );
-                    let old_base = u64::from(self.bars[bar_idx].addr & mask) << 32
-                        | u64::from(self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1]);
-                    let new_base = u64::from(value & mask) << 32
-                        | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
-                    let len = decode_64_bits_bar_size(
-                        self.bars[bar_idx].size,
-                        self.bars[bar_idx - 1].size,
-                    )
-                    .ok_or(Error::Decode64BarSize)
-                    .unwrap();
-                    let region_type = PciBarRegionType::Memory64BitRegion;
-
-                    self.bars[bar_idx].addr = value;
-                    self.bars[bar_idx - 1].addr = self.registers[reg_idx - 1];
-
-                    return Some(BarReprogrammingParams {
-                        old_base,
-                        new_base,
-                        len,
-                        region_type,
-                    });
+            // Handle special case where the address being written is
+            // different from the address initially provided. This is a
+            // BAR reprogramming case which needs to be properly caught.
+            if let Some(bar_type) = self.bars[bar_idx].r#type {
+                // In case of 64 bits memory BAR, we don't do anything until
+                // the upper BAR is modified, otherwise we would be moving the
+                // BAR to a wrong location in memory.
+                if bar_type == PciBarRegionType::Memory64BitRegion {
+                    return None;
                }
+
+                // Ignore the case where the value is unchanged.
+                if (value & mask) == (self.bars[bar_idx].addr & mask) {
+                    return None;
+                }
+
+                debug!(
+                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                    self.registers[reg_idx], value
+                );
+                let old_base = u64::from(self.bars[bar_idx].addr & mask);
+                let new_base = u64::from(value & mask);
+                let len = u64::from(
+                    decode_32_bits_bar_size(self.bars[bar_idx].size)
+                        .ok_or(Error::Decode32BarSize)
+                        .unwrap(),
+                );
+                let region_type = bar_type;
+
+                self.bars[bar_idx].addr = value;
+
+                return Some(BarReprogrammingParams {
+                    old_base,
+                    new_base,
+                    len,
+                    region_type,
+                });
+            } else if (reg_idx > BAR0_REG)
+                && ((self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
+                    != (self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1])
+                    || (value & mask) != (self.bars[bar_idx].addr & mask))
+            {
+                debug!(
+                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                    self.registers[reg_idx], value
+                );
+                let old_base = u64::from(self.bars[bar_idx].addr & mask) << 32
+                    | u64::from(self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1]);
+                let new_base = u64::from(value & mask) << 32
+                    | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
+                let len =
+                    decode_64_bits_bar_size(self.bars[bar_idx].size, self.bars[bar_idx - 1].size)
+                        .ok_or(Error::Decode64BarSize)
+                        .unwrap();
+                let region_type = PciBarRegionType::Memory64BitRegion;
+
+                self.bars[bar_idx].addr = value;
+                self.bars[bar_idx - 1].addr = self.registers[reg_idx - 1];
+
+                return Some(BarReprogrammingParams {
+                    old_base,
+                    new_base,
+                    len,
+                    region_type,
+                });
            }
        } else if reg_idx == ROM_BAR_REG && (value & mask) != (self.rom_bar_addr & mask) {
            // Ignore the case where the BAR size is being asked for.
@@ -890,43 +881,12 @@ impl Snapshottable for PciConfiguration {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut config_snapshot = Snapshot::new(self.id().as_str());
-        config_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id()),
-            snapshot,
-        });
-
-        Ok(config_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(config_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", self.id()))
-        {
-            let config_state = match serde_json::from_slice(&config_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize {}: {}",
-                        self.id(),
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&config_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find {} snapshot section",
-            self.id()
-        )))
+        self.set_state(&snapshot.to_state(&self.id())?);
+        Ok(())
    }
 }

--- a/pci/src/msix.rs
+++ b/pci/src/msix.rs
@@ -16,7 +16,7 @@ use vm_device::interrupt::{
    InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
 };
 use vm_memory::ByteValued;
-use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable};

 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
 const MSIX_TABLE_ENTRIES_MODULO: u64 = 16;
@@ -432,41 +432,18 @@ impl Snapshottable for MsixConfig {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut msix_snapshot = Snapshot::new(self.id().as_str());
-        msix_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id()),
-            snapshot,
-        });
-
-        Ok(msix_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(msix_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", self.id()))
-        {
-            let msix_state = match serde_json::from_slice(&msix_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize MSI-X {}",
-                        error
-                    )))
-                }
-            };
-
-            return self.set_state(&msix_state).map_err(|e| {
-                MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e))
-            });
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find MSI-X snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id(),
+                    e
+                ))
+            })
    }
 }

--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -910,26 +910,14 @@ impl PciDevice for VfioPciDevice {
                let first_bit = region_size.trailing_zeros();
                region_size = 2u64.pow(first_bit);

-                // We need to allocate a guest MMIO address range for that BAR.
-                // In case the BAR is mappable directly, this means it might be
-                // set as user memory region, which expects to deal with 4K
-                // pages. Therefore, the alignment has to be set accordingly.
-                let bar_alignment = if (bar_id == VFIO_PCI_ROM_REGION_INDEX)
-                    || (self.device.get_region_flags(bar_id) & VFIO_REGION_INFO_FLAG_MMAP != 0)
-                {
-                    // 4K alignment
-                    0x1000
-                } else {
-                    // Default 16 bytes alignment
-                    0x10
-                };
+                // BAR allocation needs to be naturally aligned
                if is_64bit_bar {
                    bar_addr = allocator
-                        .allocate_mmio_addresses(None, region_size, Some(bar_alignment))
+                        .allocate_mmio_addresses(None, region_size, Some(region_size))
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;
                } else {
                    bar_addr = allocator
-                        .allocate_mmio_hole_addresses(None, region_size, Some(bar_alignment))
+                        .allocate_mmio_hole_addresses(None, region_size, Some(region_size))
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;
                }
            }
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/qcow.rs"

 [dependencies]
 byteorder = "1.4.3"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 remain = "0.2.2"
 vmm-sys-util = ">=0.3.1"
--- a/rate_limiter/Cargo.toml
+++ b/rate_limiter/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "rate_limiter"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.94"
+log = "0.4.14"
+vmm-sys-util = "0.8.0"
--- a/virtio-devices/src/rate_limiter.rs
+++ b/virtio-devices/src/rate_limiter.rs
@@ -43,6 +43,9 @@
 //! It is meant to be used in an external event loop and thus implements the `AsRawFd`
 //! trait and provides an *event-handler* as part of its API. This *event-handler*
 //! needs to be called by the user on every event on the rate limiter's `AsRawFd` FD.
+#[macro_use]
+extern crate log;
+
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::time::{Duration, Instant};
 use std::{fmt, io};
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,3 +1,12 @@
+- [v15.0](#v150)
+    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
+    - [Network device rate limiting](#network-device-rate-limiting)
+    - [Support for runtime control of `virtio-net` guest offload](#support-for-runtime-control-of-virtio-net-guest-offload)
+    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
+    - [Bug fixes](#bug-fixes)
+    - [Deprecations](#deprecations)
+    - [Contributors](#contributors)
+- [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
    - [Structured event monitoring](#structured-event-monitoring)
    - [MSHV improvements](#mshv-improvements)
@@ -5,8 +14,8 @@
    - [Updated hotplug documentation](#updated-hotplug-documentation)
    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
    - [Block device rate limiting](#block-device-rate-limiting)
-    - [Deprecations](#deprecations)
-    - [Contributors](#contributors)
+    - [Deprecations](#deprecations-1)
+    - [Contributors](#contributors-1)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
    - [Improved huge page support](#improved-huge-page-support)
@@ -14,13 +23,13 @@
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-1)
+    - [Contributors](#contributors-2)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-2)
+    - [Contributors](#contributors-3)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -33,14 +42,14 @@
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Contributors](#contributors-3)
+    - [Contributors](#contributors-4)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Contributors](#contributors-4)
+    - [Contributors](#contributors-5)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -54,7 +63,7 @@
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-5)
+    - [Contributors](#contributors-6)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
@@ -63,7 +72,7 @@
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
    - [Notable Bug Fixes](#notable-bug-fixes-3)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-6)
+    - [Contributors](#contributors-7)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -73,14 +82,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-7)
+    - [Contributors](#contributors-8)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-8)
+    - [Contributors](#contributors-9)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -88,7 +97,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-9)
+    - [Contributors](#contributors-10)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -97,7 +106,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-10)
+    - [Contributors](#contributors-11)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -124,6 +133,97 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v15.0
+
+This release has been tracked through the [v15.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/18).
+
+Highlights for `cloud-hypervisor` version v15.0 include:
+
+### Version numbering and stability guarantees
+
+This release is the first in a new version numbering scheme to represent that
+we believe Cloud Hypervisor is maturing and entering a period of stability.
+With this new release we are beginning our new stability guarantees:
+
+* The API (including command line options) will not be removed or changed in a
+  breaking way without a minimum of 2 releases notice. Where possible warnings
+  will be given about the use of deprecated functionality and the deprecations
+  will be documented in the release notes.
+* Point releases will be made between individual releases where there are
+  substantial bug fixes or security issues that need to be fixed.
+
+Currently the following items are **not** guaranteed across updates:
+
+* Snapshot/restore is not supported across different versions
+* Live migration is not supported across different versions
+* The following features are considered experimental and may change
+  substantially between releases: TDX, SGX.
+
+### Network device rate limiting
+
+Building on our existing support for rate limiting block activity the network
+device also now supports rate limiting. Full details of the controls are in the
+[IO throttling documentation.](docs/io_throttling.md)
+
+### Support for runtime control of `virtio-net` guest offload
+
+The guest is now able to change the offload settings for the `virtio-net`
+device. As well as providing a useful control this mitigates an issue in the
+Linux kernel where the guest will attempt to reprogram the offload settings
+even if they are not advertised as configurable (#2528).
+
+### `--api-socket` supports file descriptor parameter
+
+The `--api-socket` can now take an `fd=` parameter to specify an existing file
+descriptor to use. This is particularly beneficial for frameworks that need to
+programmatically control Cloud Hypervisor.
+
+### Bug fixes
+
+* A workaround has been put in place to mitigate a Linux kernel issues that
+  results in the CPU thread spinning at 100% when using `virtio-pmem` (#2277).
+* PCI BARs are now correctly aligned removing the need for the guest to
+  reprogram them (#1797,#1798)
+* Handle TAP interface not being writable within virtio-net (due to the buffer
+  exhaustion on the host) (#2517)
+* The recommended Linux kernel is now v5.12.0 as it contains a fix that
+  prevents snapshot & restore working (#2535)
+
+### Deprecations
+
+Deprecated features will be removed in a subsequent release and users should plan to use alternatives
+
+* Support for booting with the "LinuxBoot" protocol for ELF and `bzImage`
+  binaries has been deprecated. When using direct boot users should configure
+  their kernel with `CONFIG_PVH=y`. Will be removed in v16.0.
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release including some new faces.
+
+* Alyssa Ross <hi@alyssa.is>
+* Anatol Belski <anbelski@linux.microsoft.com>
+* Bo Chen <chen.bo@intel.com>
+* Gaelan Steele <gbs@canishe.com>
+* Jianyong Wu <jianyong.wu@arm.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Muminul Islam <muislam@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+* William Douglas <william.douglas@intel.com>
+
+# v0.14.1
+
+Bug fix release branched off the v0.14.0 release. The following bugs were fixed
+in this release:
+
+* CPU hotplug on Windows failed due to misreported CPU state information and
+  the lack of HyperV CPUID bit enabled (#2437, #2449, #2436)
+* A seccomp rule was missing that was triggered on CPU unplug (#2455)
+* A bounds check in VIRTIO queue validation was erroneously generating
+  DescriptorChainTooShort errors in certain circumstances (#2450, #2424)
+
 # v0.14.0

 This release has been tracked through the [0.14.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/17).
@@ -165,7 +265,7 @@ PTY subsystem.

 The block device performance can now be constrained as part of the VM
 configuration allowing rate limiting. Full details of the controls are in the
-[IO throttling doumentation.](docs/io_throttling.md)
+[IO throttling documentation.](docs/io_throttling.md)


 ### Deprecations
@@ -753,11 +853,11 @@ to run their guest I/O into separate executions contexts.
 ### Command Line Interface

 More and more Cloud Hypervisor services are exposed through the
-[Rest API](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
-and thus only accessible via relatively cumbersome HTTP calls. In order
-to abstract those calls into a more user friendly tool, we created a Cloud
-Hypervisor Command Line Interface (CLI) called `ch-remote`.
-The `ch-remote` binary is created with each build and available e.g. at
+[Rest API](vmm/src/api/openapi/cloud-hypervisor.yaml) and thus only
+accessible via relatively cumbersome HTTP calls. In order to abstract
+those calls into a more user friendly tool, we created a Cloud Hypervisor
+Command Line Interface (CLI) called `ch-remote`.  The `ch-remote` binary
+is created with each build and available e.g. at
 `cloud-hypervisor/target/debug/ch-remote` when doing a debug build.

 Please check `ch-remote --help` for a complete description of all available
@@ -833,7 +933,7 @@ In order to provide a better developer experience, we worked on improving our
 build, development and testing tools.
 Somehow similar to the excellent
 [Firecracker's devtool](https://github.com/firecracker-microvm/firecracker/blob/master/tools/devtool),
-we now provide a [dev_cli script](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/scripts/dev_cli.sh).
+we now provide a [dev_cli script](scripts/dev_cli.sh).

 With this new tool, our users and contributors will be able to build and test
 Cloud Hypervisor through a containerized environment.
@@ -870,7 +970,7 @@ As a way to vertically scale Cloud-Hypervisor guests, we now support dynamically
 adding virtual CPUs to the guests, a mechanism also known as CPU hot plug.
 Through hardware-reduced ACPI notifications, Cloud Hypervisor can now add CPUs
 to an already running guest and the high level operations for that process are
-documented [here](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md)
+documented [here](docs/hotplug.md)

 During the next release cycles we are planning to extend Cloud Hypervisor
 hot plug framework to other resources, namely PCI devices and memory.
@@ -986,9 +1086,8 @@ configurations that do not require a PCI bus emulation.
 ### Paravirtualized IOMMU

 As we want to improve our nested guests support, we added support for exposing
-a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
-device through virtio. This allows for a safer nested virtio and directly
-assigned devices support.
+a [paravirtualized IOMMU](docs/iommu.md) device through virtio. This allows
+for a safer nested virtio and directly assigned devices support.

 To add the IOMMU support, we had to make some CLI changes for Cloud Hypervisor
 users to be able to specify if devices had to be handled through this virtual
@@ -1037,8 +1136,8 @@ Based on the Firecracker idea of using a dedicated I/O port to measure guest
 boot times, we added support for logging guest events through the
 [0x80](https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html)
 PC debug port. This allows, among other things, for granular guest boot time
-measurements. See our [debug port documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/debug-port.md)
-for more details.
+measurements. See our [debug port documentation](docs/debug-port.md) for more
+details.

 ### Improved direct device assignment

@@ -1073,8 +1172,8 @@ We added support for the [virtio-fs](https://virtio-fs.gitlab.io/) shared file
 system, allowing for an efficient and reliable way of sharing a filesystem
 between the host and the `cloud-hypervisor` guest.

-See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
-documentation for more details on how to use virtio-fs with `cloud-hypervisor`.
+See our [filesystem sharing](docs/fs.md) documentation for more details on how
+to use virtio-fs with `cloud-hypervisor`.

 ### Initial direct device assignment support

@@ -1082,9 +1181,8 @@ VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
 access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
 physical devices into its guest.

-See our [VFIO](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
-documentation for more detail on how to directly assign host devices to
-`cloud-hypervisor` guests.
+See our [VFIO](docs/vfio.md) documentation for more detail on how to directly
+assign host devices to `cloud-hypervisor` guests.

 ### Userspace IOAPIC

--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -1,7 +1,7 @@
-FROM ubuntu:18.04 as dev
+FROM ubuntu:20.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.49.0"
+ARG RUST_TOOLCHAIN="1.51.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -37,7 +37,6 @@ RUN apt-get update \
 	socat \
 	dosfstools \
 	cpio \
-	bsdtar \
 	libfdt-dev \
 	python3-setuptools \
    && apt-get clean \
--- a/resources/linux-config-aarch64
+++ b/resources/linux-config-aarch64
@@ -1,12 +1,13 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/arm64 5.10.0 Kernel Configuration
+# Linux/arm64 5.12.0 Kernel Configuration
 #
 CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=90300
-CONFIG_LD_VERSION=234000000
 CONFIG_CLANG_VERSION=0
+CONFIG_LD_IS_BFD=y
+CONFIG_LD_VERSION=23400
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
@@ -210,6 +211,7 @@ CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 # CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
+# CONFIG_KCMP is not set
 CONFIG_RSEQ=y
 # CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
@@ -225,7 +227,6 @@ CONFIG_PERF_EVENTS=y

 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
-# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
@@ -273,9 +274,11 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 #
 # CONFIG_ARCH_ACTIONS is not set
 # CONFIG_ARCH_AGILEX is not set
+# CONFIG_ARCH_N5X is not set
 # CONFIG_ARCH_SUNXI is not set
 # CONFIG_ARCH_ALPINE is not set
 # CONFIG_ARCH_BCM2835 is not set
+# CONFIG_ARCH_BCM4908 is not set
 # CONFIG_ARCH_BCM_IPROC is not set
 # CONFIG_ARCH_BERLIN is not set
 # CONFIG_ARCH_BITMAIN is not set
@@ -307,7 +310,6 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 # CONFIG_ARCH_VEXPRESS is not set
 # CONFIG_ARCH_VISCONTI is not set
 # CONFIG_ARCH_XGENE is not set
-# CONFIG_ARCH_ZX is not set
 # CONFIG_ARCH_ZYNQMP is not set
 # end of Platform selection

@@ -348,6 +350,7 @@ CONFIG_QCOM_FALKOR_ERRATUM_1003=y
 CONFIG_QCOM_FALKOR_ERRATUM_1009=y
 CONFIG_QCOM_QDF2400_ERRATUM_0065=y
 CONFIG_QCOM_FALKOR_ERRATUM_E1041=y
+# CONFIG_NVIDIA_CARMEL_CNP_ERRATUM is not set
 CONFIG_SOCIONEXT_SYNQUACER_PREITS=y
 # end of ARM errata workarounds via the alternatives framework

@@ -377,14 +380,11 @@ CONFIG_HZ_250=y
 # CONFIG_HZ_1000 is not set
 CONFIG_HZ=250
 CONFIG_SCHED_HRTICK=y
-CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_ARCH_SPARSEMEM_ENABLE=y
 CONFIG_ARCH_SPARSEMEM_DEFAULT=y
 CONFIG_ARCH_SELECT_MEMORY_MODEL=y
-CONFIG_HAVE_ARCH_PFN_VALID=y
 CONFIG_HW_PERF_EVENTS=y
 CONFIG_SYS_SUPPORTS_HUGETLBFS=y
-CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y
 CONFIG_PARAVIRT=y
@@ -393,6 +393,7 @@ CONFIG_PARAVIRT_TIME_ACCOUNTING=y
 CONFIG_KEXEC_FILE=y
 # CONFIG_KEXEC_SIG is not set
 # CONFIG_CRASH_DUMP is not set
+CONFIG_TRANS_TABLE=y
 # CONFIG_XEN is not set
 CONFIG_FORCE_MAX_ZONEORDER=11
 CONFIG_UNMAP_KERNEL_AT_EL0=y
@@ -406,6 +407,8 @@ CONFIG_ARM64_TAGGED_ADDR_ABI=y
 #
 CONFIG_ARM64_HW_AFDBM=y
 CONFIG_ARM64_PAN=y
+CONFIG_AS_HAS_LDAPR=y
+CONFIG_AS_HAS_LSE_ATOMICS=y
 CONFIG_ARM64_LSE_ATOMICS=y
 CONFIG_ARM64_USE_LSE_ATOMICS=y
 CONFIG_ARM64_VHE=y
@@ -414,7 +417,6 @@ CONFIG_ARM64_VHE=y
 #
 # ARMv8.2 architectural features
 #
-CONFIG_ARM64_UAO=y
 # CONFIG_ARM64_PMEM is not set
 CONFIG_ARM64_RAS_EXTN=y
 CONFIG_ARM64_CNP=y
@@ -441,6 +443,7 @@ CONFIG_ARM64_TLB_RANGE=y
 #
 # ARMv8.5 architectural features
 #
+CONFIG_AS_HAS_ARMV8_5=y
 CONFIG_ARM64_BTI=y
 CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI=y
 CONFIG_ARM64_E0PD=y
@@ -635,7 +638,6 @@ CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL=y
 CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT=y
 CONFIG_HAVE_KVM_IRQ_BYPASS=y
 CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE=y
-CONFIG_KVM_ARM_PMU=y
 # CONFIG_ARM64_CRYPTO is not set

 #
@@ -643,7 +645,6 @@ CONFIG_KVM_ARM_PMU=y
 #
 CONFIG_CRASH_CORE=y
 CONFIG_KEXEC_CORE=y
-CONFIG_SET_FS=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
@@ -679,16 +680,22 @@ CONFIG_HAVE_ARCH_SECCOMP=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
 CONFIG_SECCOMP=y
 CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
 CONFIG_HAVE_ARCH_STACKLEAK=y
 CONFIG_HAVE_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR_STRONG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG_THIN=y
+CONFIG_LTO_NONE=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PUD=y
 CONFIG_HAVE_MOVE_PMD=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
 CONFIG_HAVE_ARCH_HUGE_VMAP=y
+CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_MODULES_USE_ELF_RELA=y
 CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
 CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
@@ -707,6 +714,8 @@ CONFIG_ARCH_USE_MEMREMAP_PROT=y
 # CONFIG_LOCK_EVENT_COUNTS is not set
 CONFIG_ARCH_HAS_RELR=y
 CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
+CONFIG_HAVE_ARCH_PFN_VALID=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y

 #
 # GCOV-based kernel profiling
@@ -916,7 +925,7 @@ CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
 CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y
 CONFIG_PERCPU_STATS=y
-# CONFIG_GUP_BENCHMARK is not set
+# CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
 # end of Memory Management options
@@ -1012,6 +1021,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
+CONFIG_SOCK_RX_QUEUE_MAPPING=y
 CONFIG_XPS=y
 CONFIG_CGROUP_NET_PRIO=y
 CONFIG_CGROUP_NET_CLASSID=y
@@ -1034,7 +1044,6 @@ CONFIG_NET_FLOW_LIMIT=y
 # CONFIG_AF_KCM is not set
 CONFIG_STREAM_PARSER=y
 # CONFIG_WIRELESS is not set
-# CONFIG_WIMAX is not set
 # CONFIG_RFKILL is not set
 # CONFIG_NET_9P is not set
 # CONFIG_CAIF is not set
@@ -1071,7 +1080,6 @@ CONFIG_PCIEASPM_DEFAULT=y
 CONFIG_PCIE_PME=y
 # CONFIG_PCIE_DPC is not set
 # CONFIG_PCIE_PTM is not set
-# CONFIG_PCIE_BW is not set
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MSI_IRQ_DOMAIN=y
 CONFIG_PCI_QUIRKS=y
@@ -1105,6 +1113,7 @@ CONFIG_PCI_HOST_GENERIC=y
 # CONFIG_PCIE_ALTERA is not set
 # CONFIG_PCI_HOST_THUNDER_PEM is not set
 # CONFIG_PCI_HOST_THUNDER_ECAM is not set
+# CONFIG_PCIE_MICROCHIP_HOST is not set

 #
 # DesignWare PCI Core Support
@@ -1142,6 +1151,7 @@ CONFIG_PCI_HOST_GENERIC=y
 # CONFIG_PCI_SW_SWITCHTEC is not set
 # end of PCI switch controller drivers

+# CONFIG_CXL_BUS is not set
 # CONFIG_PCCARD is not set
 # CONFIG_RAPIDIO is not set

@@ -1174,6 +1184,7 @@ CONFIG_ALLOW_DEV_COREDUMP=y
 CONFIG_GENERIC_CPU_AUTOPROBE=y
 CONFIG_GENERIC_CPU_VULNERABILITIES=y
 CONFIG_GENERIC_ARCH_TOPOLOGY=y
+CONFIG_GENERIC_ARCH_NUMA=y
 # end of Generic Driver Options

 #
@@ -1213,6 +1224,9 @@ CONFIG_BLK_DEV=y
 CONFIG_BLK_DEV_NULL_BLK=y
 # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set
 CONFIG_ZRAM=y
+CONFIG_ZRAM_DEF_COMP_LZORLE=y
+# CONFIG_ZRAM_DEF_COMP_LZO is not set
+CONFIG_ZRAM_DEF_COMP="lzo-rle"
 # CONFIG_ZRAM_WRITEBACK is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
 # CONFIG_BLK_DEV_UMEM is not set
@@ -1221,7 +1235,6 @@ CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 CONFIG_BLK_DEV_CRYPTOLOOP=y
 # CONFIG_BLK_DEV_DRBD is not set
 # CONFIG_BLK_DEV_NBD is not set
-# CONFIG_BLK_DEV_SKD is not set
 # CONFIG_BLK_DEV_SX8 is not set
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=16
@@ -1265,6 +1278,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # Texas Instruments shared transport line discipline
 #
+# CONFIG_TI_ST is not set
 # end of Texas Instruments shared transport line discipline

 #
@@ -1272,6 +1286,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # CONFIG_GENWQE is not set
 # CONFIG_ECHO is not set
+# CONFIG_BCM_VK is not set
 # CONFIG_MISC_ALCOR_PCI is not set
 # CONFIG_MISC_RTSX_PCI is not set
 # CONFIG_HABANA_AI is not set
@@ -1324,6 +1339,7 @@ CONFIG_VIRTIO_NET=y
 #
 # Distributed Switch Architecture drivers
 #
+# CONFIG_NET_DSA_MV88E6XXX_PTP is not set
 # end of Distributed Switch Architecture drivers

 # CONFIG_ETHERNET is not set
@@ -1345,10 +1361,6 @@ CONFIG_VIRTIO_NET=y
 # Host-side USB support is needed for USB Network Adapter support
 #
 # CONFIG_WLAN is not set
-
-#
-# Enable WiMAX (Networking options) to see the WiMAX drivers
-#
 # CONFIG_WAN is not set
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
@@ -1362,7 +1374,6 @@ CONFIG_NET_FAILOVER=y
 #
 CONFIG_INPUT=y
 CONFIG_INPUT_FF_MEMLESS=y
-CONFIG_INPUT_POLLDEV=y
 CONFIG_INPUT_SPARSEKMAP=y
 # CONFIG_INPUT_MATRIXKMAP is not set

@@ -1385,6 +1396,7 @@ CONFIG_INPUT_KEYBOARD=y
 # CONFIG_KEYBOARD_LKKBD is not set
 CONFIG_KEYBOARD_GPIO=y
 CONFIG_KEYBOARD_GPIO_POLLED=y
+# CONFIG_KEYBOARD_MATRIX is not set
 # CONFIG_KEYBOARD_NEWTON is not set
 # CONFIG_KEYBOARD_OPENCORES is not set
 # CONFIG_KEYBOARD_SAMSUNG is not set
@@ -1400,9 +1412,14 @@ CONFIG_KEYBOARD_GPIO_POLLED=y
 CONFIG_INPUT_MISC=y
 # CONFIG_INPUT_AD714X is not set
 # CONFIG_INPUT_E3X0_BUTTON is not set
+# CONFIG_INPUT_GPIO_BEEPER is not set
+# CONFIG_INPUT_GPIO_DECODER is not set
+# CONFIG_INPUT_GPIO_VIBRA is not set
 CONFIG_INPUT_UINPUT=y
+# CONFIG_INPUT_GPIO_ROTARY_ENCODER is not set
 # CONFIG_INPUT_ADXL34X is not set
 # CONFIG_INPUT_CMA3000 is not set
+# CONFIG_INPUT_SOC_BUTTON_ARRAY is not set
 # CONFIG_RMI4_CORE is not set

 #
@@ -1461,6 +1478,7 @@ CONFIG_SERIAL_CORE_CONSOLE=y
 # CONFIG_SERIAL_JSM is not set
 # CONFIG_SERIAL_SIFIVE is not set
 # CONFIG_SERIAL_SCCNXP is not set
+# CONFIG_SERIAL_BCM63XX is not set
 # CONFIG_SERIAL_ALTERA_JTAGUART is not set
 # CONFIG_SERIAL_ALTERA_UART is not set
 # CONFIG_SERIAL_XILINX_PS_UART is not set
@@ -1474,6 +1492,7 @@ CONFIG_SERIAL_ARC_NR_PORTS=1
 # CONFIG_SERIAL_SPRD is not set
 # end of Serial drivers

+CONFIG_SERIAL_MCTRL_GPIO=y
 # CONFIG_SERIAL_NONSTANDARD is not set
 # CONFIG_N_GSM is not set
 # CONFIG_NOZOMI is not set
@@ -1490,7 +1509,6 @@ CONFIG_HW_RANDOM=y
 # CONFIG_HW_RANDOM_TIMERIOMEM is not set
 # CONFIG_HW_RANDOM_BA431 is not set
 CONFIG_HW_RANDOM_VIRTIO=y
-CONFIG_HW_RANDOM_HISI_V2=y
 CONFIG_HW_RANDOM_CAVIUM=y
 # CONFIG_HW_RANDOM_CCTRNG is not set
 # CONFIG_HW_RANDOM_XIPHERA is not set
@@ -1538,6 +1556,7 @@ CONFIG_PTP_1588_CLOCK=y
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
+# CONFIG_PTP_1588_CLOCK_OCP is not set
 # end of PTP clock support

 # CONFIG_PINCTRL is not set
@@ -1546,9 +1565,58 @@ CONFIG_GPIOLIB_FASTPATH_LIMIT=512
 CONFIG_OF_GPIO=y
 CONFIG_GPIO_ACPI=y
 CONFIG_GPIOLIB_IRQCHIP=y
+# CONFIG_DEBUG_GPIO is not set
+# CONFIG_GPIO_SYSFS is not set
+# CONFIG_GPIO_CDEV is not set
+
+#
+# Memory mapped GPIO drivers
+#
+# CONFIG_GPIO_74XX_MMIO is not set
+# CONFIG_GPIO_ALTERA is not set
+# CONFIG_GPIO_AMDPT is not set
+# CONFIG_GPIO_CADENCE is not set
+# CONFIG_GPIO_DWAPB is not set
+# CONFIG_GPIO_EXAR is not set
+# CONFIG_GPIO_FTGPIO010 is not set
+# CONFIG_GPIO_GENERIC_PLATFORM is not set
+# CONFIG_GPIO_GRGPIO is not set
+# CONFIG_GPIO_HISI is not set
+# CONFIG_GPIO_HLWD is not set
+# CONFIG_GPIO_MB86S7X is not set
 CONFIG_GPIO_PL061=y
+# CONFIG_GPIO_SIFIVE is not set
+# CONFIG_GPIO_XGENE is not set
+# CONFIG_GPIO_XILINX is not set
+# CONFIG_GPIO_AMD_FCH is not set
+# end of Memory mapped GPIO drivers
+
+#
+# MFD GPIO expanders
+#
+# end of MFD GPIO expanders
+
+#
+# PCI GPIO expanders
+#
+# CONFIG_GPIO_BT8XX is not set
+# CONFIG_GPIO_PCI_IDIO_16 is not set
+# CONFIG_GPIO_PCIE_IDIO_24 is not set
+# CONFIG_GPIO_RDC321X is not set
+# end of PCI GPIO expanders
+
+#
+# Virtual GPIO drivers
+#
+# CONFIG_GPIO_AGGREGATOR is not set
+# CONFIG_GPIO_MOCKUP is not set
+# end of Virtual GPIO drivers
+
 # CONFIG_W1 is not set
 CONFIG_POWER_RESET=y
+# CONFIG_POWER_RESET_GPIO is not set
+# CONFIG_POWER_RESET_GPIO_RESTART is not set
+# CONFIG_POWER_RESET_LTC2952 is not set
 # CONFIG_POWER_RESET_RESTART is not set
 # CONFIG_POWER_RESET_XGENE is not set
 # CONFIG_POWER_RESET_SYSCON is not set
@@ -1562,6 +1630,8 @@ CONFIG_POWER_SUPPLY=y
 # CONFIG_BATTERY_DS2781 is not set
 # CONFIG_BATTERY_BQ27XXX is not set
 # CONFIG_CHARGER_MAX8903 is not set
+# CONFIG_CHARGER_GPIO is not set
+# CONFIG_CHARGER_LT3651 is not set
 # CONFIG_HWMON is not set
 CONFIG_THERMAL=y
 # CONFIG_THERMAL_NETLINK is not set
@@ -1595,6 +1665,7 @@ CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # Watchdog Device Drivers
 #
 # CONFIG_SOFT_WATCHDOG is not set
+# CONFIG_GPIO_WATCHDOG is not set
 # CONFIG_WDAT_WDT is not set
 # CONFIG_XILINX_WATCHDOG is not set
 # CONFIG_ARM_SP805_WATCHDOG is not set
@@ -1605,6 +1676,7 @@ CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # CONFIG_ARM_SMC_WATCHDOG is not set
 # CONFIG_ALIM7101_WDT is not set
 # CONFIG_I6300ESB_WDT is not set
+# CONFIG_MEN_A21_WDT is not set
 CONFIG_VIRTIO_WDT=y

 #
@@ -1627,6 +1699,7 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_HTC_PASIC3 is not set
 # CONFIG_LPC_ICH is not set
 # CONFIG_LPC_SCH is not set
+# CONFIG_MFD_INTEL_PMT is not set
 # CONFIG_MFD_JANZ_CMODIO is not set
 # CONFIG_MFD_KEMPLD is not set
 # CONFIG_MFD_MT6397 is not set
@@ -1714,7 +1787,9 @@ CONFIG_FB=y
 CONFIG_LCD_CLASS_DEVICE=y
 # CONFIG_LCD_PLATFORM is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
+# CONFIG_BACKLIGHT_KTD253 is not set
 # CONFIG_BACKLIGHT_QCOM_WLED is not set
+# CONFIG_BACKLIGHT_GPIO is not set
 # end of Backlight & LCD device support

 #
@@ -1790,6 +1865,7 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PETALYNX is not set
 # CONFIG_HID_PICOLCD is not set
 # CONFIG_HID_PLANTRONICS is not set
+# CONFIG_HID_PLAYSTATION is not set
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
@@ -1883,6 +1959,7 @@ CONFIG_RTC_DRV_PL031=y
 #
 # HID Sensor RTC drivers
 #
+# CONFIG_RTC_DRV_GOLDFISH is not set
 CONFIG_DMADEVICES=y
 # CONFIG_DMADEVICES_DEBUG is not set

@@ -1924,6 +2001,7 @@ CONFIG_DMA_OF=y
 #
 # CONFIG_SYNC_FILE is not set
 # CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_DEBUG is not set
 # CONFIG_DMABUF_HEAPS is not set
 # end of DMABUF options

@@ -1949,6 +2027,7 @@ CONFIG_VFIO_PCI_INTX=y
 # CONFIG_VFIO_MDEV is not set
 # CONFIG_VIRT_DRIVERS is not set
 CONFIG_VIRTIO=y
+CONFIG_VIRTIO_PCI_LIB=y
 CONFIG_VIRTIO_MENU=y
 CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO_PCI_LEGACY=y
@@ -1973,13 +2052,15 @@ CONFIG_VHOST_MENU=y
 # CONFIG_GOLDFISH is not set
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
 CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
-# CONFIG_CLK_QORIQ is not set
+# CONFIG_COMMON_CLK_AXI_CLKGEN is not set
 # CONFIG_COMMON_CLK_XGENE is not set
 # CONFIG_COMMON_CLK_FIXED_MMIO is not set
+# CONFIG_XILINX_VCU is not set
 # CONFIG_HWSPINLOCK is not set

 #
@@ -1999,6 +2080,7 @@ CONFIG_ARM64_ERRATUM_858921=y

 CONFIG_MAILBOX=y
 # CONFIG_ARM_MHU is not set
+# CONFIG_ARM_MHU_V2 is not set
 # CONFIG_PLATFORM_MHU is not set
 # CONFIG_PL320_MBOX is not set
 CONFIG_PCC=y
@@ -2050,11 +2132,6 @@ CONFIG_ACPI_VIOT=y
 #
 # end of Amlogic SoC drivers

-#
-# Aspeed SoC drivers
-#
-# end of Aspeed SoC drivers
-
 #
 # Broadcom SoC drivers
 #
@@ -2073,6 +2150,12 @@ CONFIG_ACPI_VIOT=y
 #
 # end of i.MX SoC drivers

+#
+# Enable LiteX SoC Builder specific drivers
+#
+# CONFIG_LITEX_SOC_CONTROLLER is not set
+# end of Enable LiteX SoC Builder specific drivers
+
 #
 # Qualcomm SoC drivers
 #
@@ -2083,7 +2166,6 @@ CONFIG_ACPI_VIOT=y
 #
 # Xilinx SoC drivers
 #
-# CONFIG_XILINX_VCU is not set
 # end of Xilinx SoC drivers
 # end of SOC (System On Chip) specific Drivers

@@ -2138,8 +2220,10 @@ CONFIG_PARTITION_PERCPU=y
 # CONFIG_ARM_CMN is not set
 CONFIG_ARM_PMU=y
 CONFIG_ARM_PMU_ACPI=y
+# CONFIG_ARM_SMMU_V3_PMU is not set
 # CONFIG_ARM_DSU_PMU is not set
 # CONFIG_ARM_SPE_PMU is not set
+# CONFIG_ARM_DMC620_PMU is not set
 # CONFIG_HISI_PMU is not set
 # end of Performance monitor support

@@ -2169,6 +2253,7 @@ CONFIG_DEV_DAX_PMEM=y
 CONFIG_DEV_DAX_KMEM=y
 CONFIG_NVMEM=y
 CONFIG_NVMEM_SYSFS=y
+# CONFIG_NVMEM_RMEM is not set

 #
 # HW tracing support
@@ -2403,6 +2488,7 @@ CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR=y
 # CONFIG_HARDENED_USERCOPY is not set
 CONFIG_FORTIFY_SOURCE=y
 # CONFIG_STATIC_USERMODEHELPER is not set
+# CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT is not set
 CONFIG_DEFAULT_SECURITY_DAC=y
 CONFIG_LSM="yama,loadpin,safesetid,integrity"

@@ -2451,6 +2537,7 @@ CONFIG_CRYPTO_NULL2=y
 # CONFIG_CRYPTO_PCRYPT is not set
 CONFIG_CRYPTO_CRYPTD=y
 # CONFIG_CRYPTO_AUTHENC is not set
+# CONFIG_CRYPTO_TEST is not set

 #
 # Public-key cryptography
@@ -2510,17 +2597,13 @@ CONFIG_CRYPTO_POLY1305=y
 # CONFIG_CRYPTO_MD4 is not set
 CONFIG_CRYPTO_MD5=y
 # CONFIG_CRYPTO_MICHAEL_MIC is not set
-# CONFIG_CRYPTO_RMD128 is not set
 # CONFIG_CRYPTO_RMD160 is not set
-# CONFIG_CRYPTO_RMD256 is not set
-# CONFIG_CRYPTO_RMD320 is not set
 # CONFIG_CRYPTO_SHA1 is not set
 CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_SHA512=y
 # CONFIG_CRYPTO_SHA3 is not set
 # CONFIG_CRYPTO_SM3 is not set
 # CONFIG_CRYPTO_STREEBOG is not set
-# CONFIG_CRYPTO_TGR192 is not set
 # CONFIG_CRYPTO_WP512 is not set

 #
@@ -2534,7 +2617,6 @@ CONFIG_CRYPTO_AES=y
 # CONFIG_CRYPTO_CAST6 is not set
 CONFIG_CRYPTO_DES=y
 # CONFIG_CRYPTO_FCRYPT is not set
-# CONFIG_CRYPTO_SALSA20 is not set
 # CONFIG_CRYPTO_CHACHA20 is not set
 # CONFIG_CRYPTO_SERPENT is not set
 # CONFIG_CRYPTO_SM4 is not set
@@ -2670,6 +2752,7 @@ CONFIG_DMA_COHERENT_POOL=y
 CONFIG_DMA_REMAP=y
 CONFIG_DMA_DIRECT_REMAP=y
 # CONFIG_DMA_API_DEBUG is not set
+# CONFIG_DMA_MAP_BENCHMARK is not set
 CONFIG_SGL_ALLOC=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
@@ -2690,6 +2773,8 @@ CONFIG_SBITMAP=y
 # CONFIG_STRING_SELFTEST is not set
 # end of Library routines

+CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED=y
+
 #
 # Kernel hacking
 #
@@ -2713,7 +2798,6 @@ CONFIG_SYMBOLIC_ERRNAME=y
 # Compile-time checks and compiler options
 #
 # CONFIG_DEBUG_INFO is not set
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set
@@ -2774,9 +2858,12 @@ CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 CONFIG_HAVE_ARCH_KASAN=y
 CONFIG_HAVE_ARCH_KASAN_SW_TAGS=y
+CONFIG_HAVE_ARCH_KASAN_HW_TAGS=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
 CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 # CONFIG_KASAN is not set
+CONFIG_HAVE_ARCH_KFENCE=y
+# CONFIG_KFENCE is not set
 # end of Memory Debugging

 # CONFIG_DEBUG_SHIRQ is not set
@@ -2822,6 +2909,7 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 # CONFIG_CSD_LOCK_WAIT_DEBUG is not set
 # end of Lock Debugging (spinlocks, mutexes, etc...)

+# CONFIG_DEBUG_IRQFLAGS is not set
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
@@ -2864,7 +2952,6 @@ CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
 # CONFIG_SAMPLES is not set
-CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
 CONFIG_STRICT_DEVMEM=y
 # CONFIG_IO_STRICT_DEVMEM is not set

--- a/resources/linux-config-x86_64
+++ b/resources/linux-config-x86_64
@@ -1,12 +1,13 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 5.10.0 Kernel Configuration
+# Linux/x86 5.12.0 Kernel Configuration
 #
 CONFIG_CC_VERSION_TEXT="gcc (GCC) 10.2.1 20201125 (Red Hat 10.2.1-9)"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=100201
-CONFIG_LD_VERSION=234000000
 CONFIG_CLANG_VERSION=0
+CONFIG_LD_IS_BFD=y
+CONFIG_LD_VERSION=23500
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
@@ -235,6 +236,7 @@ CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 # CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
+# CONFIG_KCMP is not set
 CONFIG_RSEQ=y
 # CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
@@ -250,7 +252,6 @@ CONFIG_PERF_EVENTS=y

 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
-# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
@@ -289,7 +290,6 @@ CONFIG_ARCH_SUSPEND_POSSIBLE=y
 CONFIG_ARCH_WANT_GENERAL_HUGETLB=y
 CONFIG_ZONE_DMA32=y
 CONFIG_AUDIT_ARCH=y
-CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_X86_64_SMP=y
 CONFIG_ARCH_SUPPORTS_UPROBES=y
 CONFIG_FIX_EARLYCON_MEM=y
@@ -412,6 +412,7 @@ CONFIG_X86_SMAP=y
 CONFIG_X86_INTEL_TSX_MODE_OFF=y
 # CONFIG_X86_INTEL_TSX_MODE_ON is not set
 # CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
+# CONFIG_X86_SGX is not set
 CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 # CONFIG_EFI_MIXED is not set
@@ -478,6 +479,7 @@ CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
 CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
 # CONFIG_ACPI_DEBUGGER is not set
 CONFIG_ACPI_SPCR_TABLE=y
+# CONFIG_ACPI_FPDT is not set
 CONFIG_ACPI_LPIT=y
 CONFIG_ACPI_SLEEP=y
 CONFIG_ACPI_REV_OVERRIDE_POSSIBLE=y
@@ -519,7 +521,6 @@ CONFIG_HAVE_ACPI_APEI_NMI=y
 # CONFIG_ACPI_CONFIGFS is not set
 # CONFIG_PMIC_OPREGION is not set
 CONFIG_X86_PM_TIMER=y
-# CONFIG_SFI is not set

 #
 # CPU Frequency scaling
@@ -643,6 +644,7 @@ CONFIG_KVM=y
 # CONFIG_KVM_WERROR is not set
 CONFIG_KVM_INTEL=y
 # CONFIG_KVM_AMD is not set
+# CONFIG_KVM_XEN is not set
 CONFIG_AS_AVX512=y
 CONFIG_AS_SHA1_NI=y
 CONFIG_AS_SHA256_NI=y
@@ -655,8 +657,6 @@ CONFIG_CRASH_CORE=y
 CONFIG_KEXEC_CORE=y
 CONFIG_HOTPLUG_SMT=y
 CONFIG_GENERIC_ENTRY=y
-CONFIG_HAVE_OPROFILE=y
-CONFIG_OPROFILE_NMI_TIMER=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 # CONFIG_STATIC_CALL_SELFTEST is not set
@@ -701,14 +701,20 @@ CONFIG_HAVE_ARCH_SECCOMP=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
 CONFIG_SECCOMP=y
 CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
 CONFIG_HAVE_ARCH_STACKLEAK=y
 CONFIG_HAVE_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR_STRONG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG_THIN=y
+CONFIG_LTO_NONE=y
 CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
+CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PUD=y
 CONFIG_HAVE_MOVE_PMD=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD=y
@@ -717,6 +723,8 @@ CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_HAVE_ARCH_SOFT_DIRTY=y
 CONFIG_HAVE_MOD_ARCH_SPECIFIC=y
 CONFIG_MODULES_USE_ELF_RELA=y
+CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK=y
+CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK=y
 CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
 CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
 CONFIG_HAVE_EXIT_THREAD=y
@@ -735,7 +743,10 @@ CONFIG_ARCH_USE_MEMREMAP_PROT=y
 CONFIG_ARCH_HAS_MEM_ENCRYPT=y
 CONFIG_HAVE_STATIC_CALL=y
 CONFIG_HAVE_STATIC_CALL_INLINE=y
+CONFIG_HAVE_PREEMPT_DYNAMIC=y
 CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
+CONFIG_ARCH_HAS_ELFCORE_COMPAT=y

 #
 # GCOV-based kernel profiling
@@ -897,7 +908,7 @@ CONFIG_ZONE_DEVICE=y
 CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
 CONFIG_PERCPU_STATS=y
-# CONFIG_GUP_BENCHMARK is not set
+# CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
 # end of Memory Management options
@@ -993,6 +1004,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
+CONFIG_SOCK_RX_QUEUE_MAPPING=y
 CONFIG_XPS=y
 CONFIG_CGROUP_NET_PRIO=y
 CONFIG_CGROUP_NET_CLASSID=y
@@ -1015,7 +1027,6 @@ CONFIG_NET_FLOW_LIMIT=y
 # CONFIG_AF_KCM is not set
 CONFIG_STREAM_PARSER=y
 # CONFIG_WIRELESS is not set
-# CONFIG_WIMAX is not set
 # CONFIG_RFKILL is not set
 # CONFIG_NET_9P is not set
 # CONFIG_CAIF is not set
@@ -1051,7 +1062,6 @@ CONFIG_PCIEASPM_DEFAULT=y
 CONFIG_PCIE_PME=y
 # CONFIG_PCIE_DPC is not set
 # CONFIG_PCIE_PTM is not set
-# CONFIG_PCIE_BW is not set
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MSI_IRQ_DOMAIN=y
 CONFIG_PCI_QUIRKS=y
@@ -1109,6 +1119,7 @@ CONFIG_HOTPLUG_PCI_ACPI=y
 # CONFIG_PCI_SW_SWITCHTEC is not set
 # end of PCI switch controller drivers

+# CONFIG_CXL_BUS is not set
 # CONFIG_PCCARD is not set
 # CONFIG_RAPIDIO is not set

@@ -1166,6 +1177,9 @@ CONFIG_BLK_DEV=y
 CONFIG_BLK_DEV_NULL_BLK=y
 # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set
 CONFIG_ZRAM=y
+CONFIG_ZRAM_DEF_COMP_LZORLE=y
+# CONFIG_ZRAM_DEF_COMP_LZO is not set
+CONFIG_ZRAM_DEF_COMP="lzo-rle"
 # CONFIG_ZRAM_WRITEBACK is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
 # CONFIG_BLK_DEV_UMEM is not set
@@ -1174,7 +1188,6 @@ CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 CONFIG_BLK_DEV_CRYPTOLOOP=y
 # CONFIG_BLK_DEV_DRBD is not set
 # CONFIG_BLK_DEV_NBD is not set
-# CONFIG_BLK_DEV_SKD is not set
 # CONFIG_BLK_DEV_SX8 is not set
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=16
@@ -1230,6 +1243,7 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_VMWARE_VMCI is not set
 # CONFIG_GENWQE is not set
 # CONFIG_ECHO is not set
+# CONFIG_BCM_VK is not set
 # CONFIG_MISC_ALCOR_PCI is not set
 # CONFIG_MISC_RTSX_PCI is not set
 # CONFIG_HABANA_AI is not set
@@ -1285,6 +1299,7 @@ CONFIG_VIRTIO_NET=y
 #
 # Distributed Switch Architecture drivers
 #
+# CONFIG_NET_DSA_MV88E6XXX_PTP is not set
 # end of Distributed Switch Architecture drivers

 # CONFIG_ETHERNET is not set
@@ -1306,10 +1321,6 @@ CONFIG_VIRTIO_NET=y
 # Host-side USB support is needed for USB Network Adapter support
 #
 # CONFIG_WLAN is not set
-
-#
-# Enable WiMAX (Networking options) to see the WiMAX drivers
-#
 # CONFIG_WAN is not set
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
@@ -1323,7 +1334,6 @@ CONFIG_NET_FAILOVER=y
 #
 CONFIG_INPUT=y
 CONFIG_INPUT_FF_MEMLESS=y
-CONFIG_INPUT_POLLDEV=y
 CONFIG_INPUT_SPARSEKMAP=y
 # CONFIG_INPUT_MATRIXKMAP is not set

@@ -1418,6 +1428,7 @@ CONFIG_SERIAL_CORE_CONSOLE=y
 # CONFIG_SERIAL_JSM is not set
 # CONFIG_SERIAL_LANTIQ is not set
 # CONFIG_SERIAL_SCCNXP is not set
+# CONFIG_SERIAL_BCM63XX is not set
 # CONFIG_SERIAL_ALTERA_JTAGUART is not set
 # CONFIG_SERIAL_ALTERA_UART is not set
 CONFIG_SERIAL_ARC=y
@@ -1500,6 +1511,7 @@ CONFIG_PTP_1588_CLOCK=y
 #
 CONFIG_PTP_1588_CLOCK_KVM=y
 # CONFIG_PTP_1588_CLOCK_VMW is not set
+# CONFIG_PTP_1588_CLOCK_OCP is not set
 # end of PTP clock support

 # CONFIG_PINCTRL is not set
@@ -1526,6 +1538,8 @@ CONFIG_THERMAL_GOV_USER_SPACE=y
 # Intel thermal drivers
 #
 # CONFIG_INTEL_POWERCLAMP is not set
+CONFIG_X86_THERMAL_VECTOR=y
+# CONFIG_X86_PKG_TEMP_THERMAL is not set
 # CONFIG_INTEL_SOC_DTS_THERMAL is not set

 #
@@ -1614,6 +1628,7 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_LPC_SCH is not set
 # CONFIG_MFD_INTEL_LPSS_ACPI is not set
 # CONFIG_MFD_INTEL_LPSS_PCI is not set
+# CONFIG_MFD_INTEL_PMT is not set
 # CONFIG_MFD_JANZ_CMODIO is not set
 # CONFIG_MFD_KEMPLD is not set
 # CONFIG_MFD_MT6397 is not set
@@ -1787,6 +1802,7 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PETALYNX is not set
 # CONFIG_HID_PICOLCD is not set
 # CONFIG_HID_PLANTRONICS is not set
+# CONFIG_HID_PLAYSTATION is not set
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
@@ -1813,6 +1829,12 @@ CONFIG_HID_REDRAGON=y
 #
 # CONFIG_INTEL_ISH_HID is not set
 # end of Intel ISH HID support
+
+#
+# AMD SFH HID Support
+#
+# CONFIG_AMD_SFH_HID is not set
+# end of AMD SFH HID Support
 # end of HID support

 CONFIG_USB_OHCI_LITTLE_ENDIAN=y
@@ -1852,6 +1874,7 @@ CONFIG_DW_DMAC_CORE=y
 # CONFIG_DW_EDMA_PCIE is not set
 CONFIG_HSU_DMA=y
 # CONFIG_SF_PDMA is not set
+# CONFIG_INTEL_LDMA is not set

 #
 # DMA Clients
@@ -1864,6 +1887,7 @@ CONFIG_HSU_DMA=y
 #
 # CONFIG_SYNC_FILE is not set
 # CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_DEBUG is not set
 # CONFIG_DMABUF_HEAPS is not set
 # end of DMABUF options

@@ -1891,6 +1915,7 @@ CONFIG_VFIO_PCI_INTX=y
 CONFIG_IRQ_BYPASS_MANAGER=y
 # CONFIG_VIRT_DRIVERS is not set
 CONFIG_VIRTIO=y
+CONFIG_VIRTIO_PCI_LIB=y
 CONFIG_VIRTIO_MENU=y
 CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO_PCI_LEGACY=y
@@ -1915,10 +1940,12 @@ CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
 CONFIG_PMC_ATOM=y
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
 CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
+# CONFIG_XILINX_VCU is not set
 # CONFIG_HWSPINLOCK is not set

 #
@@ -1976,11 +2003,6 @@ CONFIG_ACPI_VIOT=y
 #
 # end of Amlogic SoC drivers

-#
-# Aspeed SoC drivers
-#
-# end of Aspeed SoC drivers
-
 #
 # Broadcom SoC drivers
 #
@@ -1996,6 +2018,11 @@ CONFIG_ACPI_VIOT=y
 #
 # end of i.MX SoC drivers

+#
+# Enable LiteX SoC Builder specific drivers
+#
+# end of Enable LiteX SoC Builder specific drivers
+
 #
 # Qualcomm SoC drivers
 #
@@ -2006,7 +2033,6 @@ CONFIG_ACPI_VIOT=y
 #
 # Xilinx SoC drivers
 #
-# CONFIG_XILINX_VCU is not set
 # end of Xilinx SoC drivers
 # end of SOC (System On Chip) specific Drivers

@@ -2315,8 +2341,8 @@ CONFIG_CRYPTO_NULL2=y
 # CONFIG_CRYPTO_PCRYPT is not set
 CONFIG_CRYPTO_CRYPTD=y
 # CONFIG_CRYPTO_AUTHENC is not set
+# CONFIG_CRYPTO_TEST is not set
 CONFIG_CRYPTO_SIMD=y
-CONFIG_CRYPTO_GLUE_HELPER_X86=y

 #
 # Public-key cryptography
@@ -2385,10 +2411,7 @@ CONFIG_CRYPTO_POLY1305=y
 # CONFIG_CRYPTO_MD4 is not set
 CONFIG_CRYPTO_MD5=y
 # CONFIG_CRYPTO_MICHAEL_MIC is not set
-# CONFIG_CRYPTO_RMD128 is not set
 # CONFIG_CRYPTO_RMD160 is not set
-# CONFIG_CRYPTO_RMD256 is not set
-# CONFIG_CRYPTO_RMD320 is not set
 # CONFIG_CRYPTO_SHA1 is not set
 # CONFIG_CRYPTO_SHA1_SSSE3 is not set
 # CONFIG_CRYPTO_SHA256_SSSE3 is not set
@@ -2398,7 +2421,6 @@ CONFIG_CRYPTO_SHA512=y
 # CONFIG_CRYPTO_SHA3 is not set
 # CONFIG_CRYPTO_SM3 is not set
 # CONFIG_CRYPTO_STREEBOG is not set
-# CONFIG_CRYPTO_TGR192 is not set
 # CONFIG_CRYPTO_WP512 is not set
 # CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL is not set

@@ -2421,7 +2443,6 @@ CONFIG_CRYPTO_AES_NI_INTEL=y
 CONFIG_CRYPTO_DES=y
 # CONFIG_CRYPTO_DES3_EDE_X86_64 is not set
 # CONFIG_CRYPTO_FCRYPT is not set
-# CONFIG_CRYPTO_SALSA20 is not set
 # CONFIG_CRYPTO_CHACHA20 is not set
 # CONFIG_CRYPTO_CHACHA20_X86_64 is not set
 # CONFIG_CRYPTO_SERPENT is not set
@@ -2557,6 +2578,7 @@ CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED=y
 CONFIG_SWIOTLB=y
 CONFIG_DMA_COHERENT_POOL=y
 # CONFIG_DMA_API_DEBUG is not set
+# CONFIG_DMA_MAP_BENCHMARK is not set
 CONFIG_SGL_ALLOC=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
@@ -2602,7 +2624,6 @@ CONFIG_SYMBOLIC_ERRNAME=y
 # Compile-time checks and compiler options
 #
 # CONFIG_DEBUG_INFO is not set
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set
@@ -2662,11 +2683,15 @@ CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
 # CONFIG_DEBUG_VIRTUAL is not set
 CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
+CONFIG_ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP=y
+# CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP is not set
 CONFIG_HAVE_ARCH_KASAN=y
 CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
 CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 # CONFIG_KASAN is not set
+CONFIG_HAVE_ARCH_KFENCE=y
+# CONFIG_KFENCE is not set
 # end of Memory Debugging

 # CONFIG_DEBUG_SHIRQ is not set
@@ -2714,6 +2739,7 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 # CONFIG_CSD_LOCK_WAIT_DEBUG is not set
 # end of Lock Debugging (spinlocks, mutexes, etc...)

+# CONFIG_DEBUG_IRQFLAGS is not set
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
@@ -2751,9 +2777,11 @@ CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_HAVE_FENTRY=y
+CONFIG_HAVE_OBJTOOL_MCOUNT=y
 CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -107,7 +107,7 @@ update_workloads() {
    }

    SRCDIR=$PWD
-    LINUX_CUSTOM_BRANCH="ch-5.10.6"
+    LINUX_CUSTOM_BRANCH="ch-5.12"

    # Check whether the local HEAD commit same as the remote HEAD or not. Remove the folder if they are different.
    if [ -d "$LINUX_CUSTOM_DIR" ]; then
--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -43,7 +43,7 @@ if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
 fi


-FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210106-1.qcow2"
+FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210407-0.qcow2"
 FOCAL_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
@@ -52,7 +52,7 @@ if [ ! -f "$FOCAL_OS_IMAGE" ]; then
    popd
 fi

-FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210106-1.raw"
+FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210407-0.raw"
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
    pushd $WORKLOADS_DIR
@@ -96,36 +96,26 @@ popd

 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
-VMLINUX_PVH_IMAGE="$WORKLOADS_DIR/vmlinux.pvh"
 BZIMAGE_IMAGE="$WORKLOADS_DIR/bzImage"

 LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom"

-if [ ! -f "$VMLINUX_IMAGE" ] || [ ! -f "$VMLINUX_PVH_IMAGE" ]; then
+if [ ! -f "$VMLINUX_IMAGE" ]; then
    SRCDIR=$PWD
    pushd $WORKLOADS_DIR
-    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.10.6" $LINUX_CUSTOM_DIR
+    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.12" $LINUX_CUSTOM_DIR
    cp $SRCDIR/resources/linux-config-x86_64 $LINUX_CUSTOM_DIR/.config
    popd
 fi

 if [ ! -f "$VMLINUX_IMAGE" ]; then
    pushd $LINUX_CUSTOM_DIR
-    scripts/config --disable "CONFIG_PVH"
    time make bzImage -j `nproc`
    cp vmlinux $VMLINUX_IMAGE || exit 1
    cp arch/x86/boot/bzImage $BZIMAGE_IMAGE || exit 1
    popd
 fi

-if [ ! -f "$VMLINUX_PVH_IMAGE" ]; then
-    pushd $LINUX_CUSTOM_DIR
-    scripts/config --enable "CONFIG_PVH"
-    time make bzImage -j `nproc`
-    cp vmlinux $VMLINUX_PVH_IMAGE || exit 1
-    popd
-fi
-
 if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi
--- a/scripts/sha1sums-x86_64
+++ b/scripts/sha1sums-x86_64
@@ -1,5 +1,5 @@
 27f3b17962ace69b51f0ddc2012095e3109e6ed8 bionic-server-cloudimg-amd64.qcow2
 8db9cc58b01452ce2d06c313177e6e74d8582d93 bionic-server-cloudimg-amd64.raw
 d4a44acc6014d5f83dea1c625c43d677a95fa75f alpine-minirootfs-x86_64.tar.gz
-4ee774e9e10f3cd985203c3ea20b487c285dfa1d focal-server-cloudimg-amd64-custom-20210106-1.qcow2
-7cd439f9dfe63b5f354011ea0e893256a8111174 focal-server-cloudimg-amd64-custom-20210106-1.raw
+f2f88ab4c08e36a4b6747a1aac4f94065952cadd focal-server-cloudimg-amd64-custom-20210407-0.qcow2
+6c5770ebd34c0dd53987bc9deae43690ef02ffc5 focal-server-cloudimg-amd64-custom-20210407-0.raw
--- a/src/main.rs
+++ b/src/main.rs
@@ -25,7 +25,7 @@ use signal_hook::{
 };
 use std::env;
 use std::fs::File;
-use std::os::unix::io::FromRawFd;
+use std::os::unix::io::{FromRawFd, RawFd};
 use std::sync::mpsc::channel;
 use std::sync::{Arc, Mutex};
 use std::thread;
@@ -62,6 +62,18 @@ enum Error {
    ThreadJoin(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
    #[error("VMM thread exited with error: {0}")]
    VmmThread(#[source] vmm::Error),
+    #[error("Error parsing --api-socket: {0}")]
+    ParsingApiSocket(std::num::ParseIntError),
+    #[error("Error parsing --event-monitor: {0}")]
+    ParsingEventMonitor(option_parser::OptionParserError),
+    #[error("Error parsing --event-monitor: path or fd required")]
+    BareEventMonitor,
+    #[error("Error doing event monitor I/O: {0}")]
+    EventMonitorIo(std::io::Error),
+    #[error("Error creating log file: {0}")]
+    LogFileCreation(std::io::Error),
+    #[error("Error setting up logger: {0}")]
+    LoggerSetup(log::SetLoggerError),
 }

 struct Logger {
@@ -314,7 +326,7 @@ fn create_app<'a, 'b>(
        .arg(
            Arg::with_name("api-socket")
                .long("api-socket")
-                .help("HTTP API socket path (UNIX domain socket).")
+                .help("HTTP API socket (UNIX domain socket): path=</path/to/a/file> or fd=<fd>.")
                .takes_value(true)
                .min_values(1)
                .group("vmm-config"),
@@ -369,7 +381,77 @@ fn create_app<'a, 'b>(
    app
 }

-fn start_vmm(cmd_arguments: ArgMatches, api_socket_path: &Option<String>) -> Result<(), Error> {
+fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
+    let log_level = match cmd_arguments.occurrences_of("v") {
+        0 => LevelFilter::Warn,
+        1 => LevelFilter::Info,
+        2 => LevelFilter::Debug,
+        _ => LevelFilter::Trace,
+    };
+
+    let log_file: Box<dyn std::io::Write + Send> = if let Some(file) =
+        cmd_arguments.value_of("log-file")
+    {
+        Box::new(std::fs::File::create(std::path::Path::new(file)).map_err(Error::LogFileCreation)?)
+    } else {
+        Box::new(std::io::stderr())
+    };
+
+    log::set_boxed_logger(Box::new(Logger {
+        output: Mutex::new(log_file),
+        start: std::time::Instant::now(),
+    }))
+    .map(|()| log::set_max_level(log_level))
+    .map_err(Error::LoggerSetup)?;
+
+    let (api_socket_path, api_socket_fd) =
+        if let Some(socket_config) = cmd_arguments.value_of("api-socket") {
+            let mut parser = OptionParser::new();
+            parser.add("path").add("fd");
+            parser.parse(socket_config).unwrap_or_default();
+
+            if let Some(fd) = parser.get("fd") {
+                (
+                    None,
+                    Some(fd.parse::<RawFd>().map_err(Error::ParsingApiSocket)?),
+                )
+            } else if let Some(path) = parser.get("path") {
+                (Some(path), None)
+            } else {
+                (
+                    cmd_arguments.value_of("api-socket").map(|s| s.to_string()),
+                    None,
+                )
+            }
+        } else {
+            (None, None)
+        };
+
+    if let Some(monitor_config) = cmd_arguments.value_of("event-monitor") {
+        let mut parser = OptionParser::new();
+        parser.add("path").add("fd");
+        parser
+            .parse(monitor_config)
+            .map_err(Error::ParsingEventMonitor)?;
+
+        let file = if parser.is_set("fd") {
+            let fd = parser
+                .convert("fd")
+                .map_err(Error::ParsingEventMonitor)?
+                .unwrap();
+            unsafe { File::from_raw_fd(fd) }
+        } else if parser.is_set("path") {
+            std::fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .open(parser.get("path").unwrap())
+                .map_err(Error::EventMonitorIo)?
+        } else {
+            return Err(Error::BareEventMonitor);
+        };
+        event_monitor::set_monitor(file).map_err(Error::EventMonitorIo)?;
+    }
+
    let (api_request_sender, api_request_receiver) = channel();
    let api_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateApiEventFd)?;

@@ -417,7 +499,8 @@ fn start_vmm(cmd_arguments: ArgMatches, api_socket_path: &Option<String>) -> Res
    let hypervisor = hypervisor::new().map_err(Error::CreateHypervisor)?;
    let vmm_thread = vmm::start_vmm_thread(
        env!("CARGO_PKG_VERSION").to_string(),
-        api_socket_path,
+        &api_socket_path,
+        api_socket_fd,
        api_evt.try_clone().unwrap(),
        http_sender,
        api_request_receiver,
@@ -453,7 +536,9 @@ fn start_vmm(cmd_arguments: ArgMatches, api_socket_path: &Option<String>) -> Res
    vmm_thread
        .join()
        .map_err(Error::ThreadJoin)?
-        .map_err(Error::VmmThread)
+        .map_err(Error::VmmThread)?;
+
+    Ok(api_socket_path)
 }

 fn main() {
@@ -461,67 +546,18 @@ fn main() {
    let _ = unsafe { libc::umask(0o077) };

    let (default_vcpus, default_memory, default_rng) = prepare_default_values();
-
    let cmd_arguments = create_app(&default_vcpus, &default_memory, &default_rng).get_matches();
-
-    let log_level = match cmd_arguments.occurrences_of("v") {
-        0 => LevelFilter::Warn,
-        1 => LevelFilter::Info,
-        2 => LevelFilter::Debug,
-        _ => LevelFilter::Trace,
+    let exit_code = match start_vmm(cmd_arguments) {
+        Ok(path) => {
+            path.map(|s| std::fs::remove_file(s).ok());
+            0
+        }
+        Err(e) => {
+            eprintln!("{}", e);
+            1
+        }
    };

-    let log_file: Box<dyn std::io::Write + Send> =
-        if let Some(file) = cmd_arguments.value_of("log-file") {
-            Box::new(
-                std::fs::File::create(std::path::Path::new(file)).expect("Error creating log file"),
-            )
-        } else {
-            Box::new(std::io::stderr())
-        };
-
-    log::set_boxed_logger(Box::new(Logger {
-        output: Mutex::new(log_file),
-        start: std::time::Instant::now(),
-    }))
-    .map(|()| log::set_max_level(log_level))
-    .expect("Expected to be able to setup logger");
-
-    let api_socket_path = cmd_arguments.value_of("api-socket").map(|s| s.to_string());
-
-    if let Some(monitor_config) = cmd_arguments.value_of("event-monitor") {
-        let mut parser = OptionParser::new();
-        parser.add("path").add("fd");
-        parser
-            .parse(monitor_config)
-            .expect("Error parsing event monitor config");
-
-        let file = if parser.is_set("fd") {
-            let fd = parser
-                .convert("fd")
-                .expect("Integral file descriptor expected")
-                .unwrap();
-            unsafe { File::from_raw_fd(fd) }
-        } else if parser.is_set("path") {
-            std::fs::OpenOptions::new()
-                .write(true)
-                .create(true)
-                .open(parser.get("path").unwrap())
-                .expect("Error opening event monitor path")
-        } else {
-            panic!("--event-monitor requires either a fd or path provided")
-        };
-        event_monitor::set_monitor(file).expect("Expected setting monitor to succeed");
-    }
-
-    let exit_code = if let Err(e) = start_vmm(cmd_arguments, &api_socket_path) {
-        eprintln!("{}", e);
-        1
-    } else {
-        0
-    };
-
-    api_socket_path.map(|s| std::fs::remove_file(s).ok());
    std::process::exit(exit_code);
 }

--- a/test_data/cloud-init/ubuntu/user-data
+++ b/test_data/cloud-init/ubuntu/user-data
@@ -36,14 +36,15 @@ write_files:
        #!/bin/bash

        mount /dev/vdc /mnt
+        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
        bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
-        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
+        bash -c "echo 0000:00:06.0 > /sys/bus/pci/drivers/vfio-pci/bind"
        bash -c "echo 0000:00:07.0 > /sys/bus/pci/devices/0000\:00\:07.0/driver/unbind"
-        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
+        bash -c "echo 0000:00:07.0 > /sys/bus/pci/drivers/vfio-pci/bind"
        # 1G ram requires 512 pages
        echo 512 | sudo tee /proc/sys/vm/nr_hugepages
        sudo chmod a+rwX /dev/hugepages
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210106-1.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:06.0/ path=/sys/bus/pci/devices/0000:00:07.0/ --api-socket /tmp/ch_api.sock
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210407-0.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:06.0/ path=/sys/bus/pci/devices/0000:00:07.0/ --api-socket /tmp/ch_api.sock

  -
    path: /etc/systemd/system/notify-booted.service
--- a/tests/integration.rs
+++ b/tests/integration.rs
--- a/vhost_user_backend/Cargo.toml
+++ b/vhost_user_backend/Cargo.toml
@@ -9,7 +9,7 @@ default = []

 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 virtio-bindings = "0.1.0"
 vm-memory = "0.5.0"
--- a/vhost_user_backend/src/lib.rs
+++ b/vhost_user_backend/src/lib.rs
@@ -437,9 +437,7 @@ struct VhostUserHandler<S: VhostUserBackend> {
    backend: Arc<RwLock<S>>,
    workers: Vec<Arc<VringWorker>>,
    owned: bool,
-    features_acked: bool,
    acked_features: u64,
-    acked_protocol_features: u64,
    num_queues: usize,
    max_queue_size: usize,
    queues_per_thread: Vec<u64>,
@@ -515,9 +513,7 @@ impl<S: VhostUserBackend> VhostUserHandler<S> {
            backend,
            workers,
            owned: false,
-            features_acked: false,
            acked_features: 0,
-            acked_protocol_features: 0,
            num_queues,
            max_queue_size,
            queues_per_thread,
@@ -554,9 +550,7 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandlerMut for VhostUserHandler<S> {

    fn reset_owner(&mut self) -> VhostUserResult<()> {
        self.owned = false;
-        self.features_acked = false;
        self.acked_features = 0;
-        self.acked_protocol_features = 0;
        Ok(())
    }

@@ -570,7 +564,6 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandlerMut for VhostUserHandler<S> {
        }

        self.acked_features = features;
-        self.features_acked = true;

        // If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
        // the ring is initialized in an enabled state.
@@ -597,11 +590,10 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandlerMut for VhostUserHandler<S> {
        Ok(self.backend.read().unwrap().protocol_features())
    }

-    fn set_protocol_features(&mut self, features: u64) -> VhostUserResult<()> {
+    fn set_protocol_features(&mut self, _features: u64) -> VhostUserResult<()> {
        // Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must
        // support this message even before VHOST_USER_SET_FEATURES was
        // called.
-        self.acked_protocol_features = features;
        Ok(())
    }

--- a/vhost_user_block/Cargo.toml
+++ b/vhost_user_block/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2018"
 block_util = { path = "../block_util" }
 clap = { version = "2.33.3", features=["wrap_help"] }
 epoll = ">=4.0.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 option_parser = { path = "../option_parser" }
 qcow = { path = "../qcow" }
--- a/vhost_user_block/src/lib.rs
+++ b/vhost_user_block/src/lib.rs
@@ -469,10 +469,10 @@ impl VhostUserBlkBackendConfig {
            path,
            socket,
            num_queues,
+            queue_size,
            readonly,
            direct,
            poll_queue,
-            queue_size,
        })
    }
 }
--- a/vhost_user_net/Cargo.toml
+++ b/vhost_user_net/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 clap = { version = "2.33.3", features=["wrap_help"] }
 epoll = ">=4.0.1"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
--- a/vhost_user_net/src/lib.rs
+++ b/vhost_user_net/src/lib.rs
@@ -94,7 +94,6 @@ impl std::convert::From<Error> for std::io::Error {

 struct VhostUserNetThread {
    net: NetQueuePair,
-    vring_worker: Option<Arc<VringWorker>>,
    kill_evt: EventFd,
 }

@@ -102,7 +101,6 @@ impl VhostUserNetThread {
    /// Create a new virtio network device with the given TAP interface.
    fn new(tap: Tap) -> Result<Self> {
        Ok(VhostUserNetThread {
-            vring_worker: None,
            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
            net: NetQueuePair {
                mem: None,
@@ -113,13 +111,15 @@ impl VhostUserNetThread {
                epoll_fd: None,
                counters: NetCounters::default(),
                tap_event_id: 2,
+                rx_desc_avail: false,
+                rx_rate_limiter: None,
+                tx_rate_limiter: None,
            },
        })
    }

    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
        self.net.epoll_fd = Some(vring_worker.as_ref().unwrap().as_raw_fd());
-        self.vring_worker = vring_worker;
    }
 }

@@ -176,13 +176,7 @@ impl VhostUserBackend for VhostUserNetBackend {
    }

    fn features(&self) -> u64 {
-        1 << VIRTIO_NET_F_GUEST_CSUM
-            | 1 << VIRTIO_NET_F_CSUM
-            | 1 << VIRTIO_NET_F_GUEST_TSO4
-            | 1 << VIRTIO_NET_F_GUEST_UFO
-            | 1 << VIRTIO_NET_F_HOST_TSO4
-            | 1 << VIRTIO_NET_F_HOST_UFO
-            | 1 << VIRTIO_F_VERSION_1
+        1 << VIRTIO_F_VERSION_1
            | 1 << VIRTIO_RING_F_EVENT_IDX
            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
    }
@@ -345,11 +339,7 @@ pub fn start_net_backend(backend_command: &str) {
        }
    };

-    let tap = if let Some(tap) = backend_config.tap.as_ref() {
-        Some(tap.as_str())
-    } else {
-        None
-    };
+    let tap = backend_config.tap.as_deref();

    let net_backend = Arc::new(RwLock::new(
        VhostUserNetBackend::new(
--- a/virtio-devices/Cargo.toml
+++ b/virtio-devices/Cargo.toml
@@ -16,12 +16,13 @@ byteorder = "1.4.3"
 epoll = ">=4.0.1"
 event_monitor = { path = "../event_monitor" }
 io-uring = ">=0.4.0"
-libc = "0.2.91"
+libc = "0.2.94"
 log = "0.4.14"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
+rate_limiter = { path = "../rate_limiter" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
 serde = ">=1.0.27"
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
--- a/virtio-devices/src/block.rs
+++ b/virtio-devices/src/block.rs
@@ -14,14 +14,13 @@ use super::{
    RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
    EPOLL_HELPER_EVENT_LAST,
 };
-use crate::rate_limiter::{RateLimiter, TokenType};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::VirtioInterrupt;
-use anyhow::anyhow;
 use block_util::{
    async_io::AsyncIo, async_io::AsyncIoError, async_io::DiskFile, build_disk_image_id, Request,
    RequestType, VirtioBlockConfig,
 };
+use rate_limiter::{RateLimiter, TokenType};
 use seccomp::{SeccompAction, SeccompFilter};
 use std::io;
 use std::num::Wrapping;
@@ -34,10 +33,7 @@ use std::thread;
 use std::{collections::HashMap, convert::TryInto};
 use virtio_bindings::bindings::virtio_blk::*;
 use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const SECTOR_SHIFT: u8 = 9;
@@ -369,7 +365,7 @@ pub struct Block {

 #[derive(Serialize, Deserialize)]
 pub struct BlockState {
-    pub disk_path: PathBuf,
+    pub disk_path: String,
    pub disk_nsectors: u64,
    pub avail_features: u64,
    pub acked_features: u64,
@@ -451,7 +447,7 @@ impl Block {

    fn state(&self) -> BlockState {
        BlockState {
-            disk_path: self.disk_path.clone(),
+            disk_path: self.disk_path.to_str().unwrap().to_owned(),
            disk_nsectors: self.disk_nsectors,
            avail_features: self.common.avail_features,
            acked_features: self.common.acked_features,
@@ -460,7 +456,7 @@ impl Block {
    }

    fn set_state(&mut self, state: &BlockState) {
-        self.disk_path = state.disk_path.clone();
+        self.disk_path = state.disk_path.clone().into();
        self.disk_nsectors = state.disk_nsectors;
        self.common.avail_features = state.avail_features;
        self.common.acked_features = state.acked_features;
@@ -678,37 +674,12 @@ impl Snapshottable for Block {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut block_snapshot = Snapshot::new(self.id.as_str());
-        block_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(block_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(block_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let block_state = match serde_json::from_slice(&block_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize BLOCK {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&block_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find BLOCK snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }
 impl Transportable for Block {}
--- a/virtio-devices/src/console.rs
+++ b/virtio-devices/src/console.rs
@@ -9,10 +9,8 @@ use super::{
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::VirtioInterrupt;
-use anyhow::anyhow;
 use libc::EFD_NONBLOCK;
 use seccomp::{SeccompAction, SeccompFilter};
-use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::collections::VecDeque;
 use std::io;
@@ -24,10 +22,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Barrier, Mutex};
 use std::thread;
 use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const QUEUE_SIZE: u16 = 256;
@@ -45,7 +40,7 @@ const CONFIG_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4;
 //Console size feature bit
 const VIRTIO_CONSOLE_F_SIZE: u64 = 0;

-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Deserialize, Serialize)]
 #[repr(C, packed)]
 pub struct VirtioConsoleConfig {
    cols: u16,
@@ -54,30 +49,6 @@ pub struct VirtioConsoleConfig {
    emerg_wr: u32,
 }

-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioConsoleConfig {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let cols = self.cols;
-        let rows = self.rows;
-        let max_nr_ports = self.max_nr_ports;
-        let emerg_wr = self.emerg_wr;
-
-        let mut virtio_console_config = serializer.serialize_struct("VirtioConsoleConfig", 12)?;
-        virtio_console_config.serialize_field("cols", &cols)?;
-        virtio_console_config.serialize_field("rows", &rows)?;
-        virtio_console_config.serialize_field("max_nr_ports", &max_nr_ports)?;
-        virtio_console_config.serialize_field("emerg_wr", &emerg_wr)?;
-        virtio_console_config.end()
-    }
-}
-
 // Safe because it only has data and has no implicit padding.
 unsafe impl ByteValued for VirtioConsoleConfig {}

@@ -311,7 +282,7 @@ pub struct ConsoleState {
    avail_features: u64,
    acked_features: u64,
    config: VirtioConsoleConfig,
-    in_buffer: VecDeque<u8>,
+    in_buffer: Vec<u8>,
 }

 impl Console {
@@ -366,7 +337,7 @@ impl Console {
            avail_features: self.common.avail_features,
            acked_features: self.common.acked_features,
            config: *(self.config.lock().unwrap()),
-            in_buffer: self.input.in_buffer.lock().unwrap().clone(),
+            in_buffer: self.input.in_buffer.lock().unwrap().clone().into(),
        }
    }

@@ -374,7 +345,7 @@ impl Console {
        self.common.avail_features = state.avail_features;
        self.common.acked_features = state.acked_features;
        *(self.config.lock().unwrap()) = state.config;
-        *(self.input.in_buffer.lock().unwrap()) = state.in_buffer.clone();
+        *(self.input.in_buffer.lock().unwrap()) = state.in_buffer.clone().into();
    }
 }

@@ -512,37 +483,12 @@ impl Snapshottable for Console {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut console_snapshot = Snapshot::new(self.id.as_str());
-        console_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(console_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(console_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let console_state = match serde_json::from_slice(&console_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize CONSOLE {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&console_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find CONSOLE snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }
 impl Transportable for Console {}
--- a/virtio-devices/src/iommu.rs
+++ b/virtio-devices/src/iommu.rs
@@ -10,7 +10,6 @@ use super::{
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
-use anyhow::anyhow;
 use seccomp::{SeccompAction, SeccompFilter};
 use std::collections::BTreeMap;
 use std::fmt::{self, Display};
@@ -27,10 +26,7 @@ use vm_memory::{
    Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
    GuestMemoryError, GuestMemoryMmap,
 };
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 /// Queues sizes
@@ -748,8 +744,8 @@ pub struct Iommu {
 struct IommuState {
    avail_features: u64,
    acked_features: u64,
-    endpoints: BTreeMap<u32, u32>,
-    mappings: BTreeMap<u32, BTreeMap<u64, Mapping>>,
+    endpoints: Vec<(u32, u32)>,
+    mappings: Vec<(u32, Vec<(u64, Mapping)>)>,
 }

 impl Iommu {
@@ -791,16 +787,36 @@ impl Iommu {
        IommuState {
            avail_features: self.common.avail_features,
            acked_features: self.common.acked_features,
-            endpoints: self.mapping.endpoints.read().unwrap().clone(),
-            mappings: self.mapping.mappings.read().unwrap().clone(),
+            endpoints: self
+                .mapping
+                .endpoints
+                .read()
+                .unwrap()
+                .clone()
+                .into_iter()
+                .collect(),
+            mappings: self
+                .mapping
+                .mappings
+                .read()
+                .unwrap()
+                .clone()
+                .into_iter()
+                .map(|(k, v)| (k, v.into_iter().collect()))
+                .collect(),
        }
    }

    fn set_state(&mut self, state: &IommuState) {
        self.common.avail_features = state.avail_features;
        self.common.acked_features = state.acked_features;
-        *(self.mapping.endpoints.write().unwrap()) = state.endpoints.clone();
-        *(self.mapping.mappings.write().unwrap()) = state.mappings.clone();
+        *(self.mapping.endpoints.write().unwrap()) = state.endpoints.clone().into_iter().collect();
+        *(self.mapping.mappings.write().unwrap()) = state
+            .mappings
+            .clone()
+            .into_iter()
+            .map(|(k, v)| (k, v.into_iter().collect()))
+            .collect();
    }

    // This function lets the caller specify a list of devices attached to the
@@ -975,37 +991,12 @@ impl Snapshottable for Iommu {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut iommu_snapshot = Snapshot::new(self.id.as_str());
-        iommu_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(iommu_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(iommu_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let iommu_state = match serde_json::from_slice(&iommu_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize IOMMU {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&iommu_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find IOMMU snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }
 impl Transportable for Iommu {}
--- a/virtio-devices/src/lib.rs
+++ b/virtio-devices/src/lib.rs
@@ -40,7 +40,6 @@ pub mod mem;
 pub mod net;
 pub mod net_util;
 mod pmem;
-mod rate_limiter;
 mod rng;
 pub mod seccomp_filters;
 pub mod transport;
--- a/virtio-devices/src/net.rs
+++ b/virtio-devices/src/net.rs
@@ -6,22 +6,21 @@
 // found in the THIRD-PARTY file.

 use super::net_util::{
-    build_net_config_space, build_net_config_space_with_mq, CtrlVirtio, NetCtrlEpollHandler,
-    VirtioNetConfig,
+    build_net_config_space, build_net_config_space_with_mq, virtio_features_to_tap_offload,
+    NetCtrl, NetCtrlEpollHandler, VirtioNetConfig,
 };
 use super::Error as DeviceError;
 use super::{
    ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
-    VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST,
+    RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
+    EPOLL_HELPER_EVENT_LAST,
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::VirtioInterrupt;
-use anyhow::anyhow;
 use net_util::{
    open_tap, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio, Tap, TapError, TxVirtio,
 };
 use seccomp::{SeccompAction, SeccompFilter};
-use std::collections::HashMap;
 use std::net::Ipv4Addr;
 use std::num::Wrapping;
 use std::os::unix::io::{AsRawFd, RawFd};
@@ -30,13 +29,11 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Barrier};
 use std::thread;
 use std::vec::Vec;
+use std::{collections::HashMap, convert::TryInto};
 use virtio_bindings::bindings::virtio_net::*;
 use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 // The guest has made a buffer available to receive a frame into.
@@ -45,6 +42,10 @@ pub const RX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
 pub const TX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
 // A frame is available for reading from the tap device to receive in the guest.
 pub const RX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3;
+// New 'wake up' event from the rx rate limiter
+pub const RX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4;
+// New 'wake up' event from the tx rate limiter
+pub const TX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5;

 #[derive(Debug)]
 pub enum Error {
@@ -53,6 +54,9 @@ pub enum Error {

    // Using existing tap
    TapError(TapError),
+
+    // Error calling dup() on tap fd
+    DuplicateTapFd(std::io::Error),
 }

 pub type Result<T> = result::Result<T, Error>;
@@ -87,7 +91,16 @@ impl NetEpollHandler {
            error!("Failed to get rx queue event: {:?}", e);
        }

-        if !self.net.rx_tap_listening {
+        self.net.rx_desc_avail = true;
+
+        let rate_limit_reached = self
+            .net
+            .rx_rate_limiter
+            .as_ref()
+            .map_or(false, |r| r.is_blocked());
+
+        // Start to listen on RX_TAP_EVENT only when the rate limit is not reached
+        if !self.net.rx_tap_listening && !rate_limit_reached {
            net_util::register_listener(
                self.net.epoll_fd.unwrap(),
                self.net.tap.as_raw_fd(),
@@ -101,11 +114,7 @@ impl NetEpollHandler {
        Ok(())
    }

-    fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> {
-        let queue_evt = &self.queue_evt_pair[1];
-        if let Err(e) = queue_evt.read() {
-            error!("Failed to get tx queue event: {:?}", e);
-        }
+    fn process_tx(&mut self) -> result::Result<(), DeviceError> {
        if self
            .net
            .process_tx(&mut self.queue_pair[1])
@@ -120,6 +129,25 @@ impl NetEpollHandler {
        Ok(())
    }

+    fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> {
+        let queue_evt = &self.queue_evt_pair[1];
+        if let Err(e) = queue_evt.read() {
+            error!("Failed to get tx queue event: {:?}", e);
+        }
+
+        let rate_limit_reached = self
+            .net
+            .tx_rate_limiter
+            .as_ref()
+            .map_or(false, |r| r.is_blocked());
+
+        if !rate_limit_reached {
+            self.process_tx()?;
+        }
+
+        Ok(())
+    }
+
    fn handle_rx_tap_event(&mut self) -> result::Result<(), DeviceError> {
        if self
            .net
@@ -143,6 +171,12 @@ impl NetEpollHandler {
        let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
        helper.add_event(self.queue_evt_pair[0].as_raw_fd(), RX_QUEUE_EVENT)?;
        helper.add_event(self.queue_evt_pair[1].as_raw_fd(), TX_QUEUE_EVENT)?;
+        if let Some(rate_limiter) = &self.net.rx_rate_limiter {
+            helper.add_event(rate_limiter.as_raw_fd(), RX_RATE_LIMITER_EVENT)?;
+        }
+        if let Some(rate_limiter) = &self.net.tx_rate_limiter {
+            helper.add_event(rate_limiter.as_raw_fd(), TX_RATE_LIMITER_EVENT)?;
+        }

        // If there are some already available descriptors on the RX queue,
        // then we can start the thread while listening onto the TAP.
@@ -188,6 +222,57 @@ impl EpollHelperHandler for NetEpollHandler {
                    return true;
                }
            }
+            RX_RATE_LIMITER_EVENT => {
+                if let Some(rate_limiter) = &mut self.net.rx_rate_limiter {
+                    // Upon rate limiter event, call the rate limiter handler and register the
+                    // TAP fd for further processing if some RX buffers are available
+                    match rate_limiter.event_handler() {
+                        Ok(_) => {
+                            if !self.net.rx_tap_listening && self.net.rx_desc_avail {
+                                if let Err(e) = net_util::register_listener(
+                                    self.net.epoll_fd.unwrap(),
+                                    self.net.tap.as_raw_fd(),
+                                    epoll::Events::EPOLLIN,
+                                    u64::from(self.net.tap_event_id),
+                                ) {
+                                    error!("Error register_listener with `RX_RATE_LIMITER_EVENT`: {:?}", e);
+                                    return true;
+                                }
+                                self.net.rx_tap_listening = true;
+                            }
+                        }
+                        Err(e) => {
+                            error!("Error from 'rate_limiter.event_handler()': {:?}", e);
+                            return true;
+                        }
+                    }
+                } else {
+                    error!("Unexpected RX_RATE_LIMITER_EVENT");
+                    return true;
+                }
+            }
+            TX_RATE_LIMITER_EVENT => {
+                if let Some(rate_limiter) = &mut self.net.tx_rate_limiter {
+                    // Upon rate limiter event, call the rate limiter handler
+                    // and restart processing the queue.
+                    match rate_limiter.event_handler() {
+                        Ok(_) => {
+                            self.driver_awake = true;
+                            if let Err(e) = self.process_tx() {
+                                error!("Error processing TX queue: {:?}", e);
+                                return true;
+                            }
+                        }
+                        Err(e) => {
+                            error!("Error from 'rate_limiter.event_handler()': {:?}", e);
+                            return true;
+                        }
+                    }
+                } else {
+                    error!("Unexpected TX_RATE_LIMITER_EVENT");
+                    return true;
+                }
+            }
            _ => {
                error!("Unknown event: {}", ev_type);
                return true;
@@ -200,11 +285,12 @@ impl EpollHelperHandler for NetEpollHandler {
 pub struct Net {
    common: VirtioCommon,
    id: String,
-    taps: Option<Vec<Tap>>,
+    taps: Vec<Tap>,
    config: VirtioNetConfig,
    ctrl_queue_epoll_thread: Option<thread::JoinHandle<()>>,
    counters: NetCounters,
    seccomp_action: SeccompAction,
+    rate_limiter_config: Option<RateLimiterConfig>,
 }

 #[derive(Serialize, Deserialize)]
@@ -217,6 +303,7 @@ pub struct NetState {

 impl Net {
    /// Create a new virtio network device with the given TAP interface.
+    #[allow(clippy::too_many_arguments)]
    pub fn new_with_tap(
        id: String,
        taps: Vec<Tap>,
@@ -225,12 +312,18 @@ impl Net {
        num_queues: usize,
        queue_size: u16,
        seccomp_action: SeccompAction,
+        rate_limiter_config: Option<RateLimiterConfig>,
    ) -> Result<Self> {
-        let mut avail_features = 1 << VIRTIO_NET_F_GUEST_CSUM
-            | 1 << VIRTIO_NET_F_CSUM
+        let mut avail_features = 1 << VIRTIO_NET_F_CSUM
+            | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
+            | 1 << VIRTIO_NET_F_GUEST_CSUM
+            | 1 << VIRTIO_NET_F_GUEST_ECN
            | 1 << VIRTIO_NET_F_GUEST_TSO4
+            | 1 << VIRTIO_NET_F_GUEST_TSO6
            | 1 << VIRTIO_NET_F_GUEST_UFO
+            | 1 << VIRTIO_NET_F_HOST_ECN
            | 1 << VIRTIO_NET_F_HOST_TSO4
+            | 1 << VIRTIO_NET_F_HOST_TSO6
            | 1 << VIRTIO_NET_F_HOST_UFO
            | 1 << VIRTIO_RING_F_EVENT_IDX
            | 1 << VIRTIO_F_VERSION_1;
@@ -259,11 +352,12 @@ impl Net {
                ..Default::default()
            },
            id,
-            taps: Some(taps),
+            taps,
            config,
            ctrl_queue_epoll_thread: None,
            counters: NetCounters::default(),
            seccomp_action,
+            rate_limiter_config,
        })
    }

@@ -281,6 +375,7 @@ impl Net {
        num_queues: usize,
        queue_size: u16,
        seccomp_action: SeccompAction,
+        rate_limiter_config: Option<RateLimiterConfig>,
    ) -> Result<Self> {
        let taps = open_tap(if_name, ip_addr, netmask, host_mac, num_queues / 2, None)
            .map_err(Error::OpenTap)?;
@@ -293,6 +388,7 @@ impl Net {
            num_queues,
            queue_size,
            seccomp_action,
+            rate_limiter_config,
        )
    }

@@ -303,12 +399,18 @@ impl Net {
        iommu: bool,
        queue_size: u16,
        seccomp_action: SeccompAction,
+        rate_limiter_config: Option<RateLimiterConfig>,
    ) -> Result<Self> {
        let mut taps: Vec<Tap> = Vec::new();
        let num_queue_pairs = fds.len();

        for fd in fds.iter() {
-            let tap = Tap::from_tap_fd(*fd, num_queue_pairs).map_err(Error::TapError)?;
+            // Duplicate so that it can survive reboots
+            let fd = unsafe { libc::dup(*fd) };
+            if fd < 0 {
+                return Err(Error::DuplicateTapFd(std::io::Error::last_os_error()));
+            }
+            let tap = Tap::from_tap_fd(fd, num_queue_pairs).map_err(Error::TapError)?;
            taps.push(tap);
        }

@@ -320,6 +422,7 @@ impl Net {
            num_queue_pairs * 2,
            queue_size,
            seccomp_action,
+            rate_limiter_config,
        )
    }

@@ -377,152 +480,171 @@ impl VirtioDevice for Net {
        mut queues: Vec<Queue>,
        mut queue_evts: Vec<EventFd>,
    ) -> ActivateResult {
-        if let Some(mut taps) = self.taps.clone() {
-            self.common.activate(&queues, &queue_evts, &interrupt_cb)?;
+        self.common.activate(&queues, &queue_evts, &interrupt_cb)?;

-            let queue_num = queues.len();
-            if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && queue_num % 2 != 0 {
-                let cvq_queue = queues.remove(queue_num - 1);
-                let cvq_queue_evt = queue_evts.remove(queue_num - 1);
+        let queue_num = queues.len();
+        if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && queue_num % 2 != 0 {
+            let cvq_queue = queues.remove(queue_num - 1);
+            let cvq_queue_evt = queue_evts.remove(queue_num - 1);

-                let kill_evt = self
-                    .common
-                    .kill_evt
-                    .as_ref()
-                    .unwrap()
-                    .try_clone()
-                    .map_err(|e| {
-                        error!("failed to clone kill_evt eventfd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
-                let pause_evt = self
-                    .common
-                    .pause_evt
-                    .as_ref()
-                    .unwrap()
-                    .try_clone()
-                    .map_err(|e| {
-                        error!("failed to clone pause_evt eventfd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
+            let kill_evt = self
+                .common
+                .kill_evt
+                .as_ref()
+                .unwrap()
+                .try_clone()
+                .map_err(|e| {
+                    error!("failed to clone kill_evt eventfd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+            let pause_evt = self
+                .common
+                .pause_evt
+                .as_ref()
+                .unwrap()
+                .try_clone()
+                .map_err(|e| {
+                    error!("failed to clone pause_evt eventfd: {}", e);
+                    ActivateError::BadActivate
+                })?;

-                let mut ctrl_handler = NetCtrlEpollHandler {
-                    mem: mem.clone(),
-                    kill_evt,
-                    pause_evt,
-                    ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
-                    epoll_fd: 0,
-                };
+            let mut ctrl_handler = NetCtrlEpollHandler {
+                mem: mem.clone(),
+                kill_evt,
+                pause_evt,
+                ctrl_q: NetCtrl::new(cvq_queue, cvq_queue_evt, Some(self.taps.clone())),
+            };

-                let paused = self.common.paused.clone();
-                // Let's update the barrier as we need 1 for each RX/TX pair +
-                // 1 for the control queue + 1 for the main thread signalling
-                // the pause.
-                self.common.paused_sync = Some(Arc::new(Barrier::new(taps.len() + 2)));
-                let paused_sync = self.common.paused_sync.clone();
+            let paused = self.common.paused.clone();
+            // Let's update the barrier as we need 1 for each RX/TX pair +
+            // 1 for the control queue + 1 for the main thread signalling
+            // the pause.
+            self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2)));
+            let paused_sync = self.common.paused_sync.clone();

-                // Retrieve seccomp filter for virtio_net_ctl thread
-                let virtio_net_ctl_seccomp_filter =
-                    get_seccomp_filter(&self.seccomp_action, Thread::VirtioNetCtl)
-                        .map_err(ActivateError::CreateSeccompFilter)?;
-                thread::Builder::new()
-                    .name(format!("{}_ctrl", self.id))
-                    .spawn(move || {
-                        if let Err(e) = SeccompFilter::apply(virtio_net_ctl_seccomp_filter) {
-                            error!("Error applying seccomp filter: {:?}", e);
-                        } else if let Err(e) = ctrl_handler.run_ctrl(paused, paused_sync.unwrap()) {
-                            error!("Error running worker: {:?}", e);
-                        }
-                    })
-                    .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
-                    .map_err(|e| {
-                        error!("failed to clone queue EventFd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
-            }
-
-            let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into());
-
-            let mut epoll_threads = Vec::new();
-            for i in 0..taps.len() {
-                let rx = RxVirtio::new();
-                let tx = TxVirtio::new();
-                let rx_tap_listening = false;
-
-                let mut queue_pair = vec![queues.remove(0), queues.remove(0)];
-                queue_pair[0].set_event_idx(event_idx);
-                queue_pair[1].set_event_idx(event_idx);
-
-                let queue_evt_pair = vec![queue_evts.remove(0), queue_evts.remove(0)];
-
-                let kill_evt = self
-                    .common
-                    .kill_evt
-                    .as_ref()
-                    .unwrap()
-                    .try_clone()
-                    .map_err(|e| {
-                        error!("failed to clone kill_evt eventfd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
-                let pause_evt = self
-                    .common
-                    .pause_evt
-                    .as_ref()
-                    .unwrap()
-                    .try_clone()
-                    .map_err(|e| {
-                        error!("failed to clone pause_evt eventfd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
-
-                let mut handler = NetEpollHandler {
-                    net: NetQueuePair {
-                        mem: Some(mem.clone()),
-                        tap: taps.remove(0),
-                        rx,
-                        tx,
-                        epoll_fd: None,
-                        rx_tap_listening,
-                        counters: self.counters.clone(),
-                        tap_event_id: RX_TAP_EVENT,
-                    },
-                    queue_pair,
-                    queue_evt_pair,
-                    interrupt_cb: interrupt_cb.clone(),
-                    kill_evt,
-                    pause_evt,
-                    driver_awake: false,
-                };
-
-                let paused = self.common.paused.clone();
-                let paused_sync = self.common.paused_sync.clone();
-                // Retrieve seccomp filter for virtio_net thread
-                let virtio_net_seccomp_filter =
-                    get_seccomp_filter(&self.seccomp_action, Thread::VirtioNet)
-                        .map_err(ActivateError::CreateSeccompFilter)?;
-                thread::Builder::new()
-                    .name(format!("{}_qp{}", self.id.clone(), i))
-                    .spawn(move || {
-                        if let Err(e) = SeccompFilter::apply(virtio_net_seccomp_filter) {
-                            error!("Error applying seccomp filter: {:?}", e);
-                        } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
-                            error!("Error running worker: {:?}", e);
-                        }
-                    })
-                    .map(|thread| epoll_threads.push(thread))
-                    .map_err(|e| {
-                        error!("failed to clone queue EventFd: {}", e);
-                        ActivateError::BadActivate
-                    })?;
-            }
-
-            self.common.epoll_threads = Some(epoll_threads);
-
-            event!("virtio-device", "activated", "id", &self.id);
-            return Ok(());
+            // Retrieve seccomp filter for virtio_net_ctl thread
+            let virtio_net_ctl_seccomp_filter =
+                get_seccomp_filter(&self.seccomp_action, Thread::VirtioNetCtl)
+                    .map_err(ActivateError::CreateSeccompFilter)?;
+            thread::Builder::new()
+                .name(format!("{}_ctrl", self.id))
+                .spawn(move || {
+                    if let Err(e) = SeccompFilter::apply(virtio_net_ctl_seccomp_filter) {
+                        error!("Error applying seccomp filter: {:?}", e);
+                    } else if let Err(e) = ctrl_handler.run_ctrl(paused, paused_sync.unwrap()) {
+                        error!("Error running worker: {:?}", e);
+                    }
+                })
+                .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
        }
-        Err(ActivateError::BadActivate)
+
+        let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into());
+
+        let mut epoll_threads = Vec::new();
+        let mut taps = self.taps.clone();
+        for i in 0..taps.len() {
+            let rx = RxVirtio::new();
+            let tx = TxVirtio::new();
+            let rx_tap_listening = false;
+
+            let mut queue_pair = vec![queues.remove(0), queues.remove(0)];
+            queue_pair[0].set_event_idx(event_idx);
+            queue_pair[1].set_event_idx(event_idx);
+
+            let queue_evt_pair = vec![queue_evts.remove(0), queue_evts.remove(0)];
+
+            let kill_evt = self
+                .common
+                .kill_evt
+                .as_ref()
+                .unwrap()
+                .try_clone()
+                .map_err(|e| {
+                    error!("failed to clone kill_evt eventfd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+            let pause_evt = self
+                .common
+                .pause_evt
+                .as_ref()
+                .unwrap()
+                .try_clone()
+                .map_err(|e| {
+                    error!("failed to clone pause_evt eventfd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+
+            let rx_rate_limiter: Option<rate_limiter::RateLimiter> = self
+                .rate_limiter_config
+                .map(RateLimiterConfig::try_into)
+                .transpose()
+                .map_err(ActivateError::CreateRateLimiter)?;
+
+            let tx_rate_limiter: Option<rate_limiter::RateLimiter> = self
+                .rate_limiter_config
+                .map(RateLimiterConfig::try_into)
+                .transpose()
+                .map_err(ActivateError::CreateRateLimiter)?;
+
+            let tap = taps.remove(0);
+            tap.set_offload(virtio_features_to_tap_offload(self.common.acked_features))
+                .map_err(|e| {
+                    error!("Error programming tap offload: {:?}", e);
+                    ActivateError::BadActivate
+                })?;
+
+            let mut handler = NetEpollHandler {
+                net: NetQueuePair {
+                    mem: Some(mem.clone()),
+                    tap,
+                    rx,
+                    tx,
+                    epoll_fd: None,
+                    rx_tap_listening,
+                    counters: self.counters.clone(),
+                    tap_event_id: RX_TAP_EVENT,
+                    rx_desc_avail: false,
+                    rx_rate_limiter,
+                    tx_rate_limiter,
+                },
+                queue_pair,
+                queue_evt_pair,
+                interrupt_cb: interrupt_cb.clone(),
+                kill_evt,
+                pause_evt,
+                driver_awake: false,
+            };
+
+            let paused = self.common.paused.clone();
+            let paused_sync = self.common.paused_sync.clone();
+            // Retrieve seccomp filter for virtio_net thread
+            let virtio_net_seccomp_filter =
+                get_seccomp_filter(&self.seccomp_action, Thread::VirtioNet)
+                    .map_err(ActivateError::CreateSeccompFilter)?;
+            thread::Builder::new()
+                .name(format!("{}_qp{}", self.id.clone(), i))
+                .spawn(move || {
+                    if let Err(e) = SeccompFilter::apply(virtio_net_seccomp_filter) {
+                        error!("Error applying seccomp filter: {:?}", e);
+                    } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+                        error!("Error running worker: {:?}", e);
+                    }
+                })
+                .map(|thread| epoll_threads.push(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+
+        self.common.epoll_threads = Some(epoll_threads);
+
+        event!("virtio-device", "activated", "id", &self.id);
+        Ok(())
    }

    fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
@@ -576,37 +698,12 @@ impl Snapshottable for Net {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut net_snapshot = Snapshot::new(self.id.as_str());
-        net_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(net_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(net_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let net_state = match serde_json::from_slice(&net_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize NET {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&net_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find NET snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }
 impl Transportable for Net {}
--- a/virtio-devices/src/net_util.rs
+++ b/virtio-devices/src/net_util.rs
@@ -2,13 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause

-use super::{
-    DescriptorChain, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
-    EPOLL_HELPER_EVENT_LAST,
-};
+use super::{EpollHelper, EpollHelperError, EpollHelperHandler, Queue, EPOLL_HELPER_EVENT_LAST};
 use net_util::MacAddr;
-use serde::ser::{Serialize, SerializeStruct, Serializer};
-use std::os::unix::io::{AsRawFd, RawFd};
+use net_util::Tap;
+use std::os::raw::c_uint;
+use std::os::unix::io::AsRawFd;
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier};
 use virtio_bindings::bindings::virtio_net::*;
@@ -25,7 +23,7 @@ const QUEUE_SIZE: usize = 256;
 const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;

 #[repr(C, packed)]
-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Deserialize, Serialize)]
 pub struct VirtioNetConfig {
    pub mac: [u8; 6],
    pub status: u16,
@@ -35,126 +33,111 @@ pub struct VirtioNetConfig {
    pub duplex: u8,
 }

-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioNetConfig {
-    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mac = self.mac;
-        let status = self.status;
-        let max_virtqueue_pairs = self.max_virtqueue_pairs;
-        let mtu = self.mtu;
-        let speed = self.speed;
-        let duplex = self.duplex;
-
-        let mut virtio_net_config = serializer.serialize_struct("VirtioNetConfig", 17)?;
-        virtio_net_config.serialize_field("mac", &mac)?;
-        virtio_net_config.serialize_field("status", &status)?;
-        virtio_net_config.serialize_field("max_virtqueue_pairs", &max_virtqueue_pairs)?;
-        virtio_net_config.serialize_field("mtu", &mtu)?;
-        virtio_net_config.serialize_field("speed", &speed)?;
-        virtio_net_config.serialize_field("duplex", &duplex)?;
-        virtio_net_config.end()
-    }
-}
-
 // Safe because it only has data and has no implicit padding.
 unsafe impl ByteValued for VirtioNetConfig {}

 #[derive(Debug)]
 pub enum Error {
-    /// Read process MQ.
-    FailedProcessMq,
    /// Read queue failed.
    GuestMemory(GuestMemoryError),
-    /// Invalid ctrl class
-    InvalidCtlClass,
-    /// Invalid ctrl command
-    InvalidCtlCmd,
-    /// Invalid descriptor
-    InvalidDesc,
-    /// Invalid queue pairs number
-    InvalidQueuePairsNum,
-    /// No memory passed in.
-    NoMemory,
-    /// No ueue pairs number.
-    NoQueuePairsNum,
+    /// No queue pairs number.
+    NoQueuePairsDescriptor,
+    /// No status descriptor
+    NoStatusDescriptor,
 }

-pub struct CtrlVirtio {
+pub struct NetCtrl {
    pub queue_evt: EventFd,
    pub queue: Queue,
+    pub taps: Option<Vec<Tap>>,
 }

-impl std::clone::Clone for CtrlVirtio {
-    fn clone(&self) -> Self {
-        CtrlVirtio {
-            queue_evt: self.queue_evt.try_clone().unwrap(),
-            queue: self.queue.clone(),
-        }
-    }
+#[repr(C, packed)]
+#[derive(Debug, Clone, Copy, Default)]
+pub struct ControlHeader {
+    pub class: u8,
+    pub cmd: u8,
 }

-impl CtrlVirtio {
-    pub fn new(queue: Queue, queue_evt: EventFd) -> Self {
-        CtrlVirtio { queue_evt, queue }
-    }
+unsafe impl ByteValued for ControlHeader {}

-    fn process_mq(&self, mem: &GuestMemoryMmap, avail_desc: DescriptorChain) -> Result<()> {
-        let mq_desc = if avail_desc.has_next() {
-            avail_desc.next_descriptor().unwrap()
-        } else {
-            return Err(Error::NoQueuePairsNum);
-        };
-        let queue_pairs = mem
-            .read_obj::<u16>(mq_desc.addr)
-            .map_err(Error::GuestMemory)?;
-        if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
-            || (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
-        {
-            return Err(Error::InvalidQueuePairsNum);
+impl NetCtrl {
+    pub fn new(queue: Queue, queue_evt: EventFd, taps: Option<Vec<Tap>>) -> Self {
+        NetCtrl {
+            queue_evt,
+            queue,
+            taps,
        }
-        let status_desc = if mq_desc.has_next() {
-            mq_desc.next_descriptor().unwrap()
-        } else {
-            return Err(Error::NoQueuePairsNum);
-        };
-        mem.write_obj::<u8>(0, status_desc.addr)
-            .map_err(Error::GuestMemory)?;
-
-        Ok(())
    }

    pub fn process_cvq(&mut self, mem: &GuestMemoryMmap) -> Result<()> {
        let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
        let mut used_count = 0;
-        if let Some(avail_desc) = self.queue.iter(&mem).next() {
-            used_desc_heads[used_count] = (avail_desc.index, avail_desc.len);
-            used_count += 1;
-            let ctrl_hdr = mem
-                .read_obj::<u16>(avail_desc.addr)
-                .map_err(Error::GuestMemory)?;
-            let ctrl_hdr_v = ctrl_hdr.as_slice();
-            let class = ctrl_hdr_v[0];
-            let cmd = ctrl_hdr_v[1];
-            match u32::from(class) {
+        let queue = &mut self.queue;
+        for avail_desc in queue.iter(&mem) {
+            let ctrl_hdr: ControlHeader =
+                mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?;
+            let data_desc = avail_desc
+                .next_descriptor()
+                .ok_or(Error::NoQueuePairsDescriptor)?;
+            let status_desc = data_desc
+                .next_descriptor()
+                .ok_or(Error::NoStatusDescriptor)?;
+
+            let ok = match u32::from(ctrl_hdr.class) {
                VIRTIO_NET_CTRL_MQ => {
-                    if u32::from(cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
-                        return Err(Error::InvalidCtlCmd);
-                    }
-                    if let Err(_e) = self.process_mq(&mem, avail_desc) {
-                        return Err(Error::FailedProcessMq);
+                    let queue_pairs = mem
+                        .read_obj::<u16>(data_desc.addr)
+                        .map_err(Error::GuestMemory)?;
+                    if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
+                        warn!("Unsupported command: {}", ctrl_hdr.cmd);
+                        false
+                    } else if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
+                        || (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
+                    {
+                        warn!("Number of MQ pairs out of range: {}", queue_pairs);
+                        false
+                    } else {
+                        info!("Number of MQ pairs requested: {}", queue_pairs);
+                        true
                    }
                }
-                _ => return Err(Error::InvalidCtlClass),
-            }
-        } else {
-            return Err(Error::InvalidDesc);
+                VIRTIO_NET_CTRL_GUEST_OFFLOADS => {
+                    let features = mem
+                        .read_obj::<u64>(data_desc.addr)
+                        .map_err(Error::GuestMemory)?;
+                    if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET {
+                        warn!("Unsupported command: {}", ctrl_hdr.cmd);
+                        false
+                    } else {
+                        let mut ok = true;
+                        if let Some(ref mut taps) = &mut self.taps {
+                            for tap in taps.iter_mut() {
+                                info!("Reprogramming tap offload with features: {}", features);
+                                tap.set_offload(virtio_features_to_tap_offload(features))
+                                    .map_err(|e| {
+                                        error!("Error programming tap offload: {:?}", e);
+                                        ok = false
+                                    })
+                                    .ok();
+                            }
+                        }
+                        ok
+                    }
+                }
+                _ => {
+                    warn!("Unsupported command {:?}", ctrl_hdr);
+                    false
+                }
+            };
+
+            mem.write_obj(
+                if ok { VIRTIO_NET_OK } else { VIRTIO_NET_ERR } as u8,
+                status_desc.addr,
+            )
+            .map_err(Error::GuestMemory)?;
+            used_desc_heads[used_count] = (avail_desc.index, avail_desc.len);
+            used_count += 1;
        }
        for &(desc_index, len) in &used_desc_heads[..used_count] {
            self.queue.add_used(&mem, desc_index, len);
@@ -169,8 +152,7 @@ pub struct NetCtrlEpollHandler {
    pub mem: GuestMemoryAtomic<GuestMemoryMmap>,
    pub kill_evt: EventFd,
    pub pause_evt: EventFd,
-    pub ctrl_q: CtrlVirtio,
-    pub epoll_fd: RawFd,
+    pub ctrl_q: NetCtrl,
 }

 impl NetCtrlEpollHandler {
@@ -237,3 +219,24 @@ pub fn build_net_config_space_with_mq(
        *avail_features |= 1 << VIRTIO_NET_F_MQ;
    }
 }
+
+pub fn virtio_features_to_tap_offload(features: u64) -> c_uint {
+    let mut tap_offloads: c_uint = 0;
+    if features & (1 << VIRTIO_NET_F_GUEST_CSUM) != 0 {
+        tap_offloads |= net_gen::TUN_F_CSUM;
+    }
+    if features & (1 << VIRTIO_NET_F_GUEST_TSO4) != 0 {
+        tap_offloads |= net_gen::TUN_F_TSO4;
+    }
+    if features & (1 << VIRTIO_NET_F_GUEST_TSO6) != 0 {
+        tap_offloads |= net_gen::TUN_F_TSO6;
+    }
+    if features & (1 << VIRTIO_NET_F_GUEST_ECN) != 0 {
+        tap_offloads |= net_gen::TUN_F_TSO_ECN;
+    }
+    if features & (1 << VIRTIO_NET_F_GUEST_UFO) != 0 {
+        tap_offloads |= net_gen::TUN_F_UFO;
+    }
+
+    tap_offloads
+}
--- a/virtio-devices/src/pmem.rs
+++ b/virtio-devices/src/pmem.rs
@@ -14,7 +14,6 @@ use super::{
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::{VirtioInterrupt, VirtioInterruptType};
-use anyhow::anyhow;
 use seccomp::{SeccompAction, SeccompFilter};
 use std::fmt::{self, Display};
 use std::fs::File;
@@ -29,10 +28,7 @@ use vm_memory::{
    Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
    GuestMemoryError, GuestMemoryMmap, MmapRegion,
 };
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const QUEUE_SIZE: u16 = 256;
@@ -461,37 +457,12 @@ impl Snapshottable for Pmem {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut pmem_snapshot = Snapshot::new(self.id.as_str());
-        pmem_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(pmem_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(pmem_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let pmem_state = match serde_json::from_slice(&pmem_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize PMEM {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&pmem_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find PMEM snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/virtio-devices/src/rng.rs
+++ b/virtio-devices/src/rng.rs
@@ -10,7 +10,6 @@ use super::{
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::{VirtioInterrupt, VirtioInterruptType};
-use anyhow::anyhow;
 use seccomp::{SeccompAction, SeccompFilter};
 use std::fs::File;
 use std::io;
@@ -20,10 +19,7 @@ use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier};
 use std::thread;
 use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const QUEUE_SIZE: u16 = 256;
@@ -303,37 +299,12 @@ impl Snapshottable for Rng {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut rng_snapshot = Snapshot::new(self.id.as_str());
-        rng_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(rng_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(rng_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let rng_state = match serde_json::from_slice(&rng_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize RNG {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&rng_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find RNG snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/virtio-devices/src/seccomp_filters.rs
+++ b/virtio-devices/src/seccomp_filters.rs
@@ -59,6 +59,9 @@ const FIONBIO: u64 = 0x5421;
 const VFIO_IOMMU_MAP_DMA: u64 = 0x3b71;
 const VFIO_IOMMU_UNMAP_DMA: u64 = 0x3b72;

+// See include/uapi/linux/if_tun.h in the kernel code.
+const TUNSETOFFLOAD: u64 = 0x4004_54d0;
+
 fn create_virtio_iommu_ioctl_seccomp_rule() -> Vec<SeccompRule> {
    or![
        and![Cond::new(1, ArgLen::DWORD, Eq, VFIO_IOMMU_MAP_DMA).unwrap()],
@@ -228,13 +231,18 @@ fn virtio_net_thread_rules() -> Vec<SyscallRuleSet> {
        allow_syscall(libc::SYS_readv),
        allow_syscall(libc::SYS_rt_sigprocmask),
        allow_syscall(libc::SYS_sigaltstack),
+        allow_syscall(libc::SYS_timerfd_settime),
        allow_syscall(libc::SYS_write),
        allow_syscall(libc::SYS_writev),
    ]
 }

-fn virtio_net_ctl_thread_rules() -> Vec<SyscallRuleSet> {
-    vec![
+fn create_virtio_net_ctl_ioctl_seccomp_rule() -> Result<Vec<SeccompRule>, Error> {
+    Ok(or![and![Cond::new(1, ArgLen::DWORD, Eq, TUNSETOFFLOAD)?],])
+}
+
+fn virtio_net_ctl_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
+    Ok(vec![
        allow_syscall(libc::SYS_brk),
        allow_syscall(libc::SYS_close),
        allow_syscall(libc::SYS_dup),
@@ -245,13 +253,14 @@ fn virtio_net_ctl_thread_rules() -> Vec<SyscallRuleSet> {
        allow_syscall(libc::SYS_epoll_wait),
        allow_syscall(libc::SYS_exit),
        allow_syscall(libc::SYS_futex),
+        allow_syscall_if(libc::SYS_ioctl, create_virtio_net_ctl_ioctl_seccomp_rule()?),
        allow_syscall(libc::SYS_madvise),
        allow_syscall(libc::SYS_munmap),
        allow_syscall(libc::SYS_read),
        allow_syscall(libc::SYS_rt_sigprocmask),
        allow_syscall(libc::SYS_sigaltstack),
        allow_syscall(libc::SYS_write),
-    ]
+    ])
 }

 fn virtio_pmem_thread_rules() -> Vec<SyscallRuleSet> {
@@ -377,12 +386,13 @@ fn virtio_vhost_net_ctl_thread_rules() -> Vec<SyscallRuleSet> {
        allow_syscall(libc::SYS_epoll_pwait),
        #[cfg(target_arch = "x86_64")]
        allow_syscall(libc::SYS_epoll_wait),
+        allow_syscall(libc::SYS_exit),
        allow_syscall(libc::SYS_futex),
-        allow_syscall(libc::SYS_read),
-        allow_syscall(libc::SYS_sigaltstack),
        allow_syscall(libc::SYS_munmap),
        allow_syscall(libc::SYS_madvise),
-        allow_syscall(libc::SYS_exit),
+        allow_syscall(libc::SYS_read),
+        allow_syscall(libc::SYS_sigaltstack),
+        allow_syscall(libc::SYS_write),
    ]
 }

@@ -449,7 +459,7 @@ fn get_seccomp_filter_trap(thread_type: Thread) -> Result<SeccompFilter, Error>
        Thread::VirtioIommu => virtio_iommu_thread_rules(),
        Thread::VirtioMem => virtio_mem_thread_rules(),
        Thread::VirtioNet => virtio_net_thread_rules(),
-        Thread::VirtioNetCtl => virtio_net_ctl_thread_rules(),
+        Thread::VirtioNetCtl => virtio_net_ctl_thread_rules()?,
        Thread::VirtioPmem => virtio_pmem_thread_rules(),
        Thread::VirtioRng => virtio_rng_thread_rules(),
        Thread::VirtioVhostBlk => virtio_vhost_blk_thread_rules(),
@@ -471,7 +481,7 @@ fn get_seccomp_filter_log(thread_type: Thread) -> Result<SeccompFilter, Error> {
        Thread::VirtioIommu => virtio_iommu_thread_rules(),
        Thread::VirtioMem => virtio_mem_thread_rules(),
        Thread::VirtioNet => virtio_net_thread_rules(),
-        Thread::VirtioNetCtl => virtio_net_ctl_thread_rules(),
+        Thread::VirtioNetCtl => virtio_net_ctl_thread_rules()?,
        Thread::VirtioPmem => virtio_pmem_thread_rules(),
        Thread::VirtioRng => virtio_rng_thread_rules(),
        Thread::VirtioVhostBlk => virtio_vhost_blk_thread_rules(),
--- a/virtio-devices/src/transport/pci_common_config.rs
+++ b/virtio-devices/src/transport/pci_common_config.rs
@@ -8,12 +8,11 @@
 extern crate byteorder;

 use crate::{Queue, VirtioDevice};
-use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::{Arc, Mutex};
 use vm_memory::GuestAddress;
-use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable};

 #[derive(Clone, Serialize, Deserialize)]
 pub struct VirtioPciCommonConfigState {
@@ -291,43 +290,12 @@ impl Snapshottable for VirtioPciCommonConfig {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut config_snapshot = Snapshot::new(self.id().as_str());
-        config_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id()),
-            snapshot,
-        });
-
-        Ok(config_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(config_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", self.id()))
-        {
-            let config_state = match serde_json::from_slice(&config_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize {}: {}",
-                        self.id(),
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&config_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find {} snapshot section",
-            self.id()
-        )))
+        self.set_state(&snapshot.to_state(&self.id())?);
+        Ok(())
    }
 }

--- a/virtio-devices/src/transport/pci_device.rs
+++ b/virtio-devices/src/transport/pci_device.rs
@@ -41,10 +41,7 @@ use vm_memory::{
    Address, ByteValued, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap,
    GuestUsize, Le32,
 };
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vm_virtio::{queue, VirtioIommuRemapping, VIRTIO_MSI_NO_VECTOR};
 use vmm_sys_util::{errno::Result, eventfd::EventFd};

@@ -835,14 +832,22 @@ impl PciDevice for VirtioPciDevice {
        let (virtio_pci_bar_addr, region_type) = if self.use_64bit_bar {
            let region_type = PciBarRegionType::Memory64BitRegion;
            let addr = allocator
-                .allocate_mmio_addresses(self.settings_bar_addr, CAPABILITY_BAR_SIZE, None)
+                .allocate_mmio_addresses(
+                    self.settings_bar_addr,
+                    CAPABILITY_BAR_SIZE,
+                    Some(CAPABILITY_BAR_SIZE),
+                )
                .ok_or(PciDeviceError::IoAllocationFailed(CAPABILITY_BAR_SIZE))?;
            ranges.push((addr, CAPABILITY_BAR_SIZE, region_type));
            (addr, region_type)
        } else {
            let region_type = PciBarRegionType::Memory32BitRegion;
            let addr = allocator
-                .allocate_mmio_hole_addresses(self.settings_bar_addr, CAPABILITY_BAR_SIZE, None)
+                .allocate_mmio_hole_addresses(
+                    self.settings_bar_addr,
+                    CAPABILITY_BAR_SIZE,
+                    Some(CAPABILITY_BAR_SIZE),
+                )
                .ok_or(PciDeviceError::IoAllocationFailed(CAPABILITY_BAR_SIZE))?;
            ranges.push((addr, CAPABILITY_BAR_SIZE, region_type));
            (addr, region_type)
@@ -1078,14 +1083,7 @@ impl Snapshottable for VirtioPciDevice {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut virtio_pci_dev_snapshot = Snapshot::new(self.id.as_str());
-        virtio_pci_dev_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
+        let mut virtio_pci_dev_snapshot = Snapshot::new_from_state(&self.id, &self.state())?;

        // Snapshot PciConfiguration
        virtio_pci_dev_snapshot.add_snapshot(self.configuration.snapshot()?);
@@ -1127,24 +1125,14 @@ impl Snapshottable for VirtioPciDevice {
                self.configuration.restore(*pci_config_snapshot.clone())?;
            }

-            let virtio_pci_dev_state =
-                match serde_json::from_slice(&virtio_pci_dev_section.snapshot) {
-                    Ok(state) => state,
-                    Err(error) => {
-                        return Err(MigratableError::Restore(anyhow!(
-                            "Could not deserialize VIRTIO_PCI_DEVICE {}",
-                            error
-                        )))
-                    }
-                };
-
            // First restore the status of the virtqueues.
-            self.set_state(&virtio_pci_dev_state).map_err(|e| {
-                MigratableError::Restore(anyhow!(
-                    "Could not restore VIRTIO_PCI_DEVICE state {:?}",
-                    e
-                ))
-            })?;
+            self.set_state(&virtio_pci_dev_section.to_state()?)
+                .map_err(|e| {
+                    MigratableError::Restore(anyhow!(
+                        "Could not restore VIRTIO_PCI_DEVICE state {:?}",
+                        e
+                    ))
+                })?;

            // Then we can activate the device, as we know at this point that
            // the virtqueues are in the right state and the device is ready
--- a/virtio-devices/src/vhost_user/fs.rs
+++ b/virtio-devices/src/vhost_user/fs.rs
@@ -538,11 +538,7 @@ impl VirtioDevice for Fs {
    }

    fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
-        if let Some(cache) = self.cache.as_ref() {
-            Some(cache.0.clone())
-        } else {
-            None
-        }
+        self.cache.as_ref().map(|cache| cache.0.clone())
    }

    fn set_shm_regions(
--- a/virtio-devices/src/vhost_user/net.rs
+++ b/virtio-devices/src/vhost_user/net.rs
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0

 use super::super::net_util::{
-    build_net_config_space, CtrlVirtio, NetCtrlEpollHandler, VirtioNetConfig,
+    build_net_config_space, NetCtrl, NetCtrlEpollHandler, VirtioNetConfig,
 };
 use super::super::{
    ActivateError, ActivateResult, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType,
@@ -247,8 +247,7 @@ impl VirtioDevice for Net {
                mem: mem.clone(),
                kill_evt,
                pause_evt,
-                ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
-                epoll_fd: 0,
+                ctrl_q: NetCtrl::new(cvq_queue, cvq_queue_evt, None),
            };

            let paused = self.common.paused.clone();
--- a/virtio-devices/src/vsock/device.rs
+++ b/virtio-devices/src/vsock/device.rs
@@ -17,7 +17,6 @@ use crate::{
    VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST,
    VIRTIO_F_IN_ORDER, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
-use anyhow::anyhow;
 /// This is the `VirtioDevice` implementation for our vsock device. It handles the virtio-level
 /// device logic: feature negotiation, device configuration, and device activation.
 /// The run-time device logic (i.e. event-driven data handling) is implemented by
@@ -47,10 +46,7 @@ use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier, RwLock};
 use std::thread;
 use vm_memory::{GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const QUEUE_SIZE: u16 = 256;
@@ -508,37 +504,12 @@ where
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut vsock_snapshot = Snapshot::new(self.id.as_str());
-        vsock_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(vsock_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(vsock_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let vsock_state = match serde_json::from_slice(&vsock_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize VSOCK {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&vsock_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find VSOCK snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }
 impl<B> Transportable for Vsock<B> where B: VsockBackend + Sync + 'static {}
--- a/virtio-devices/src/watchdog.rs
+++ b/virtio-devices/src/watchdog.rs
@@ -23,10 +23,7 @@ use std::sync::{Arc, Barrier, Mutex};
 use std::thread;
 use std::time::Instant;
 use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 const QUEUE_SIZE: u16 = 8;
@@ -395,38 +392,12 @@ impl Snapshottable for Watchdog {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut watchdog_snapshot = Snapshot::new(self.id.as_str());
-        watchdog_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(watchdog_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(watchdog_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id))
-        {
-            let watchdog_state = match serde_json::from_slice(&watchdog_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize watchdog {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&watchdog_state);
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find watchdog snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/vm-allocator/Cargo.toml
+++ b/vm-allocator/Cargo.toml
@@ -5,6 +5,6 @@ authors = ["The Chromium OS Authors"]
 edition = "2018"

 [dependencies]
-libc = "0.2.91"
+libc = "0.2.94"
 vm-memory = "0.5.0"
 arch = { path = "../arch" }
--- a/vm-device/src/bus.rs
+++ b/vm-device/src/bus.rs
@@ -24,8 +24,6 @@ pub trait BusDevice: Send {
    fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
        None
    }
-    /// Triggers the `irq_mask` interrupt on this device
-    fn interrupt(&self, irq_mask: u32) {}
 }

 #[derive(Debug)]
--- a/vm-migration/src/lib.rs
+++ b/vm-migration/src/lib.rs
@@ -9,6 +9,8 @@ extern crate thiserror;
 extern crate serde_derive;
 extern crate vm_memory;

+use anyhow::anyhow;
+use serde::{Deserialize, Serialize};
 use thiserror::Error;

 pub mod protocol;
@@ -64,6 +66,34 @@ pub struct SnapshotDataSection {
    pub snapshot: Vec<u8>,
 }

+impl SnapshotDataSection {
+    /// Generate the state data from the snapshot data
+    pub fn to_state<'a, T>(&'a self) -> Result<T, MigratableError>
+    where
+        T: Deserialize<'a>,
+    {
+        serde_json::from_slice(&self.snapshot).map_err(|e| {
+            MigratableError::Restore(anyhow!("Error deserialising: {} {}", self.id, e))
+        })
+    }
+
+    /// Create from state that can be serialized
+    pub fn new_from_state<T>(id: &str, state: &T) -> Result<Self, MigratableError>
+    where
+        T: Serialize,
+    {
+        let snapshot = serde_json::to_vec(state)
+            .map_err(|e| MigratableError::Snapshot(anyhow!("Error serialising: {} {}", id, e)))?;
+
+        let snapshot_data = SnapshotDataSection {
+            id: format!("{}-section", id),
+            snapshot,
+        };
+
+        Ok(snapshot_data)
+    }
+}
+
 /// A Snapshottable component's snapshot is a tree of snapshots, where leafs
 /// contain the snapshot data. Nodes of this tree track all their children
 /// through the snapshots field, which is basically their sub-components.
@@ -97,6 +127,17 @@ impl Snapshot {
        }
    }

+    /// Create from state that can be serialized
+    pub fn new_from_state<T>(id: &str, state: &T) -> Result<Self, MigratableError>
+    where
+        T: Serialize,
+    {
+        let mut snapshot_data = Snapshot::new(id);
+        snapshot_data.add_data_section(SnapshotDataSection::new_from_state(id, state)?);
+
+        Ok(snapshot_data)
+    }
+
    /// Add a sub-component's Snapshot to the Snapshot.
    pub fn add_snapshot(&mut self, snapshot: Snapshot) {
        self.snapshots
@@ -107,6 +148,17 @@ impl Snapshot {
    pub fn add_data_section(&mut self, section: SnapshotDataSection) {
        self.snapshot_data.insert(section.id.clone(), section);
    }
+
+    /// Generate the state data from the snapshot
+    pub fn to_state<'a, T>(&'a self, id: &str) -> Result<T, MigratableError>
+    where
+        T: Deserialize<'a>,
+    {
+        self.snapshot_data
+            .get(&format!("{}-section", id))
+            .ok_or_else(|| MigratableError::Restore(anyhow!("Missing section for {}", id)))?
+            .to_state()
+    }
 }

 /// A snapshottable component can be snapshotted.
--- a/vm-virtio/src/queue.rs
+++ b/vm-virtio/src/queue.rs
@@ -265,10 +265,7 @@ impl<'a> DescriptorChain<'a> {
    }

    fn is_valid(&self) -> bool {
-        !(self
-            .mem
-            .checked_offset(self.addr, self.len as usize)
-            .is_none()
+        !(!self.mem.check_range(self.addr, self.len as usize)
            || (self.has_next() && self.next >= self.table_size))
    }

--- a/vmm/Cargo.toml
+++ b/vmm/Cargo.toml
@@ -27,19 +27,19 @@ epoll = ">=4.0.1"
 event_monitor = { path = "../event_monitor" }
 hypervisor = { path = "../hypervisor" }
 lazy_static = "1.4.0"
-libc = "0.2.91"
+libc = "0.2.94"
 linux-loader = { version = "0.3.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.14"
-micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "master" }
+micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
 pci = { path = "../pci" }
 qcow = { path = "../qcow" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
-signal-hook = "0.3.7"
+signal-hook = "0.3.8"
 thiserror = "1.0"
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "master" }
 virtio-devices = { path = "../virtio-devices" }
--- a/vmm/src/api/http.rs
+++ b/vmm/src/api/http.rs
@@ -11,6 +11,8 @@ use micro_http::{Body, HttpServer, MediaType, Method, Request, Response, StatusC
 use seccomp::{SeccompAction, SeccompFilter};
 use serde_json::Error as SerdeError;
 use std::collections::HashMap;
+use std::os::unix::io::{IntoRawFd, RawFd};
+use std::os::unix::net::UnixListener;
 use std::path::PathBuf;
 use std::sync::mpsc::Sender;
 use std::sync::Arc;
@@ -253,16 +255,12 @@ fn handle_http_request(
    response
 }

-pub fn start_http_thread(
-    path: &str,
+fn start_http_thread(
+    mut server: HttpServer,
    api_notifier: EventFd,
    api_sender: Sender<ApiRequest>,
    seccomp_action: &SeccompAction,
 ) -> Result<thread::JoinHandle<Result<()>>> {
-    std::fs::remove_file(path).unwrap_or_default();
-    let socket_path = PathBuf::from(path);
-    let mut server = HttpServer::new(socket_path).map_err(Error::CreateApiServer)?;
-
    // Retrieve seccomp filter for API thread
    let api_seccomp_filter =
        get_seccomp_filter(seccomp_action, Thread::Api).map_err(Error::CreateSeccompFilter)?;
@@ -299,3 +297,27 @@ pub fn start_http_thread(
        })
        .map_err(Error::HttpThreadSpawn)
 }
+
+pub fn start_http_path_thread(
+    path: &str,
+    api_notifier: EventFd,
+    api_sender: Sender<ApiRequest>,
+    seccomp_action: &SeccompAction,
+) -> Result<thread::JoinHandle<Result<()>>> {
+    std::fs::remove_file(path).unwrap_or_default();
+    let socket_path = PathBuf::from(path);
+    let socket_fd = UnixListener::bind(socket_path).map_err(Error::CreateApiServerSocket)?;
+    let server =
+        HttpServer::new_from_fd(socket_fd.into_raw_fd()).map_err(Error::CreateApiServer)?;
+    start_http_thread(server, api_notifier, api_sender, seccomp_action)
+}
+
+pub fn start_http_fd_thread(
+    fd: RawFd,
+    api_notifier: EventFd,
+    api_sender: Sender<ApiRequest>,
+    seccomp_action: &SeccompAction,
+) -> Result<thread::JoinHandle<Result<()>>> {
+    let server = HttpServer::new_from_fd(fd).map_err(Error::CreateApiServer)?;
+    start_http_thread(server, api_notifier, api_sender, seccomp_action)
+}
--- a/vmm/src/api/mod.rs
+++ b/vmm/src/api/mod.rs
@@ -31,7 +31,8 @@
 extern crate vm_device;
 extern crate vmm_sys_util;

-pub use self::http::start_http_thread;
+pub use self::http::start_http_fd_thread;
+pub use self::http::start_http_path_thread;

 pub mod http;
 pub mod http_endpoint;
--- a/vmm/src/api/openapi/cloud-hypervisor.yaml
+++ b/vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -719,6 +719,8 @@ components:
          items:
            type: integer
            format: int32
+        rate_limiter_config:
+            $ref: '#/components/schemas/RateLimiterConfig'

    RngConfig:
      required:
--- a/vmm/src/config.rs
+++ b/vmm/src/config.rs
@@ -117,6 +117,8 @@ pub enum ValidationError {
    VnetQueueLowerThan2,
    /// The input queue number for virtio_net must match the number of input fds
    VnetQueueFdMismatch,
+    /// Using reserved fd
+    VnetReservedFd,
    // Hugepages not turned on
    HugePageSizeWithoutHugePages,
    // Huge page size is not power of 2
@@ -153,6 +155,7 @@ impl fmt::Display for ValidationError {
                f,
                "Number of queues to virtio_net does not match the number of input FDs"
            ),
+            VnetReservedFd => write!(f, "Reserved fd number (<= 2)"),
            HugePageSizeWithoutHugePages => {
                write!(f, "Huge page size specified but huge pages not enabled")
            }
@@ -852,12 +855,12 @@ impl DiskConfig {
            iommu,
            num_queues,
            queue_size,
-            vhost_socket,
            vhost_user,
+            vhost_socket,
            poll_queue,
+            rate_limiter_config,
            id,
            disable_io_uring,
-            rate_limiter_config,
        })
    }
 }
@@ -887,6 +890,8 @@ pub struct NetConfig {
    pub id: Option<String>,
    #[serde(default)]
    pub fds: Option<Vec<i32>>,
+    #[serde(default)]
+    pub rate_limiter_config: Option<RateLimiterConfig>,
 }

 fn default_netconfig_tap() -> Option<String> {
@@ -928,6 +933,7 @@ impl Default for NetConfig {
            vhost_socket: None,
            id: None,
            fds: None,
+            rate_limiter_config: None,
        }
    }
 }
@@ -936,7 +942,9 @@ impl NetConfig {
    pub const SYNTAX: &'static str = "Network parameters \
    \"tap=<if_name>,ip=<ip_addr>,mask=<net_mask>,mac=<mac_addr>,fd=<fd1:fd2...>,iommu=on|off,\
    num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,\
-    vhost_user=<vhost_user_enable>,socket=<vhost_user_socket_path>,id=<device_id>\"";
+    vhost_user=<vhost_user_enable>,socket=<vhost_user_socket_path>,id=<device_id>,\
+    bw_size=<bytes>,bw_one_time_burst=<bytes>,bw_refill_time=<ms>,\
+    ops_size=<io_ops>,ops_one_time_burst=<io_ops>,ops_refill_time=<ms>\"";

    pub fn parse(net: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
@@ -953,7 +961,13 @@ impl NetConfig {
            .add("vhost_user")
            .add("socket")
            .add("id")
-            .add("fd");
+            .add("fd")
+            .add("bw_size")
+            .add("bw_one_time_burst")
+            .add("bw_refill_time")
+            .add("ops_size")
+            .add("ops_one_time_burst")
+            .add("ops_refill_time");
        parser.parse(net).map_err(Error::ParseNetwork)?;

        let tap = parser.get("tap");
@@ -995,6 +1009,57 @@ impl NetConfig {
            .map_err(Error::ParseNetwork)?
            .map(|v| v.0.iter().map(|e| *e as i32).collect());

+        let bw_size = parser
+            .convert("bw_size")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let bw_one_time_burst = parser
+            .convert("bw_one_time_burst")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let bw_refill_time = parser
+            .convert("bw_refill_time")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let ops_size = parser
+            .convert("ops_size")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let ops_one_time_burst = parser
+            .convert("ops_one_time_burst")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let ops_refill_time = parser
+            .convert("ops_refill_time")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or_default();
+        let bw_tb_config = if bw_size != 0 && bw_refill_time != 0 {
+            Some(TokenBucketConfig {
+                size: bw_size,
+                one_time_burst: Some(bw_one_time_burst),
+                refill_time: bw_refill_time,
+            })
+        } else {
+            None
+        };
+        let ops_tb_config = if ops_size != 0 && ops_refill_time != 0 {
+            Some(TokenBucketConfig {
+                size: ops_size,
+                one_time_burst: Some(ops_one_time_burst),
+                refill_time: ops_refill_time,
+            })
+        } else {
+            None
+        };
+        let rate_limiter_config = if bw_tb_config.is_some() || ops_tb_config.is_some() {
+            Some(RateLimiterConfig {
+                bandwidth: bw_tb_config,
+                ops: ops_tb_config,
+            })
+        } else {
+            None
+        };
+
        let config = NetConfig {
            tap,
            ip,
@@ -1008,10 +1073,12 @@ impl NetConfig {
            vhost_socket,
            id,
            fds,
+            rate_limiter_config,
        };
        config.validate().map_err(Error::Validation)?;
        Ok(config)
    }
+
    pub fn validate(&self) -> ValidationResult<()> {
        if self.num_queues < 2 {
            return Err(ValidationError::VnetQueueLowerThan2);
@@ -1021,6 +1088,14 @@ impl NetConfig {
            return Err(ValidationError::VnetQueueFdMismatch);
        }

+        if let Some(fds) = self.fds.as_ref() {
+            for fd in fds {
+                if *fd <= 2 {
+                    return Err(ValidationError::VnetReservedFd);
+                }
+            }
+        }
+
        Ok(())
    }
 }
@@ -1319,7 +1394,7 @@ impl ConsoleConfig {
            .unwrap_or(Toggle(false))
            .0;

-        Ok(Self { mode, file, iommu })
+        Ok(Self { file, mode, iommu })
    }

    pub fn default_serial() -> Self {
@@ -1652,6 +1727,7 @@ impl VmConfig {
                if net.vhost_user && !self.memory.shared {
                    return Err(ValidationError::VhostUserRequiresSharedMemory);
                }
+                net.validate()?;
            }
        }

@@ -2531,6 +2607,13 @@ mod tests {
        still_valid_config.memory.shared = true;
        assert!(still_valid_config.validate().is_ok());

+        let mut invalid_config = valid_config.clone();
+        invalid_config.net = Some(vec![NetConfig {
+            fds: Some(vec![0]),
+            ..Default::default()
+        }]);
+        assert!(invalid_config.validate().is_err());
+
        let mut invalid_config = valid_config.clone();
        invalid_config.fs = Some(vec![FsConfig {
            ..Default::default()
--- a/vmm/src/cpu.rs
+++ b/vmm/src/cpu.rs
@@ -59,6 +59,10 @@ const HYPERVISOR_ECX_BIT: u8 = 31; // Hypervisor ecx bit.
 #[cfg(target_arch = "x86_64")]
 const MTRR_EDX_BIT: u8 = 12; // Hypervisor ecx bit.

+// KVM feature bits
+#[cfg(target_arch = "x86_64")]
+const KVM_FEATURE_ASYNC_PF_INT_BIT: u8 = 14;
+
 #[cfg(feature = "acpi")]
 pub const CPU_MANAGER_ACPI_SIZE: usize = 0xc;

@@ -361,41 +365,18 @@ impl Snapshottable for Vcpu {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot = serde_json::to_vec(&self.saved_state)
-            .map_err(|e| MigratableError::Snapshot(e.into()))?;
-
        let mut vcpu_snapshot = Snapshot::new(&format!("{}", self.id));
-        vcpu_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", VCPU_SNAPSHOT_ID),
-            snapshot,
-        });
+        vcpu_snapshot.add_data_section(SnapshotDataSection::new_from_state(
+            VCPU_SNAPSHOT_ID,
+            &self.saved_state,
+        )?);

        Ok(vcpu_snapshot)
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(vcpu_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", VCPU_SNAPSHOT_ID))
-        {
-            let vcpu_state = match serde_json::from_slice(&vcpu_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize the vCPU snapshot {}",
-                        error
-                    )))
-                }
-            };
-
-            self.saved_state = Some(vcpu_state);
-
-            Ok(())
-        } else {
-            Err(MigratableError::Restore(anyhow!(
-                "Could not find the vCPU snapshot section"
-            )))
-        }
+        self.saved_state = Some(snapshot.to_state(VCPU_SNAPSHOT_ID)?);
+        Ok(())
    }
 }

@@ -433,12 +414,16 @@ const CPU_SELECTION_OFFSET: u64 = 0;

 impl BusDevice for CpuManager {
    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        // The Linux kernel, quite reasonably, doesn't zero the memory it gives us.
+        data.copy_from_slice(&[0; 8][0..data.len()]);
+
        match offset {
+            CPU_SELECTION_OFFSET => {
+                data[0] = self.selected_cpu;
+            }
            CPU_STATUS_OFFSET => {
                if self.selected_cpu < self.present_vcpus() {
                    let state = &self.vcpu_states[usize::from(self.selected_cpu)];
-                    // The Linux kernel, quite reasonably, doesn't zero the memory it gives us.
-                    data.copy_from_slice(&[0; 8][0..data.len()]);
                    if state.active() {
                        data[0] |= 1 << CPU_ENABLE_FLAG;
                    }
@@ -560,12 +545,12 @@ impl CpuManager {
        vcpu_states.resize_with(usize::from(config.max_vcpus), VcpuState::default);

        #[cfg(target_arch = "x86_64")]
-        let sgx_epc_sections =
-            if let Some(sgx_epc_region) = memory_manager.lock().unwrap().sgx_epc_region() {
-                Some(sgx_epc_region.epc_sections().clone())
-            } else {
-                None
-            };
+        let sgx_epc_sections = memory_manager
+            .lock()
+            .unwrap()
+            .sgx_epc_region()
+            .as_ref()
+            .map(|sgx_epc_region| sgx_epc_region.epc_sections().clone());
        #[cfg(target_arch = "x86_64")]
        let cpuid = {
            let phys_bits = physical_bits(config.max_phys_bits);
@@ -681,10 +666,22 @@ impl CpuManager {
                .map_err(Error::CpuidSgx)?;
        }

-        // Set CPU physical bits
+        // Update some existing CPUID
        for entry in cpuid.as_mut_slice().iter_mut() {
-            if entry.function == 0x8000_0008 {
-                entry.eax = (entry.eax & 0xffff_ff00) | (phys_bits as u32 & 0xff);
+            match entry.function {
+                // Set CPU physical bits
+                0x8000_0008 => {
+                    entry.eax = (entry.eax & 0xffff_ff00) | (phys_bits as u32 & 0xff);
+                }
+                // Disable KVM_FEATURE_ASYNC_PF_INT
+                // This is required until we find out why the asynchronous page
+                // fault is generating unexpected behavior when using interrupt
+                // mechanism.
+                // TODO: Re-enable KVM_FEATURE_ASYNC_PF_INT (#2277)
+                0x4000_0001 => {
+                    entry.eax &= !(1 << KVM_FEATURE_ASYNC_PF_INT_BIT);
+                }
+                _ => {}
            }
        }

@@ -742,10 +739,18 @@ impl CpuManager {
                       | 1 << 2 // AccessSynicRegs
                       | 1 << 3 // AccessSyntheticTimerRegs
                       | 1 << 9, // AccessPartitionReferenceTsc
+                    edx: 1 << 3, // CPU dynamic partitioning
                    ..Default::default()
                })
                .map_err(Error::CpuidKvmHyperV)?;
-            for i in 0x4000_0004..=0x4000_000a {
+            cpuid
+                .push(CpuIdEntry {
+                    function: 0x4000_0004,
+                    eax: 1 << 5, // Recommend relaxed timing
+                    ..Default::default()
+                })
+                .map_err(Error::CpuidKvmHyperV)?;
+            for i in 0x4000_0005..=0x4000_000a {
                cpuid
                    .push(CpuIdEntry {
                        function: i,
@@ -850,12 +855,7 @@ impl CpuManager {
            .map_err(Error::CreateSeccompFilter)?;

        #[cfg(target_arch = "x86_64")]
-        let interrupt_controller_clone =
-            if let Some(interrupt_controller) = &self.interrupt_controller {
-                Some(interrupt_controller.clone())
-            } else {
-                None
-            };
+        let interrupt_controller_clone = self.interrupt_controller.as_ref().cloned();

        let handle = Some(
            thread::Builder::new()
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -30,14 +30,12 @@ use acpi_tables::{aml, aml::Aml};
 use anyhow::anyhow;
 #[cfg(target_arch = "aarch64")]
 use arch::aarch64::gic::GicDevice;
-#[cfg(target_arch = "aarch64")]
-use arch::aarch64::DeviceInfoForFdt;
 #[cfg(feature = "acpi")]
 use arch::layout;
 #[cfg(target_arch = "x86_64")]
 use arch::layout::{APIC_START, IOAPIC_SIZE, IOAPIC_START};
 #[cfg(target_arch = "aarch64")]
-use arch::DeviceType;
+use arch::{DeviceType, MmioDeviceInfo};
 use block_util::{
    async_io::DiskFile, block_io_uring_is_supported, detect_image_type,
    fixed_vhd_async::FixedVhdDiskAsync, fixed_vhd_sync::FixedVhdDiskSync, qcow_sync::QcowDiskSync,
@@ -786,28 +784,6 @@ struct DeviceManagerState {
    device_id_cnt: Wrapping<usize>,
 }

-/// Private structure for storing information about the MMIO device registered at some address on the bus.
-#[derive(Clone, Debug)]
-#[cfg(target_arch = "aarch64")]
-pub struct MmioDeviceInfo {
-    addr: u64,
-    irq: u32,
-    len: u64,
-}
-
-#[cfg(target_arch = "aarch64")]
-impl DeviceInfoForFdt for MmioDeviceInfo {
-    fn addr(&self) -> u64 {
-        self.addr
-    }
-    fn irq(&self) -> u32 {
-        self.irq
-    }
-    fn length(&self) -> u64 {
-        self.len
-    }
-}
-
 #[derive(Debug)]
 pub struct PtyPair {
    pub main: File,
@@ -886,6 +862,7 @@ pub struct DeviceManager {
    // MSI Interrupt Manager
    msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,

+    #[cfg_attr(feature = "mshv", allow(dead_code))]
    // Legacy Interrupt Manager
    legacy_interrupt_manager: Option<Arc<dyn InterruptManager<GroupConfig = LegacyIrqGroupConfig>>>,

@@ -1552,7 +1529,6 @@ impl DeviceManager {
            (DeviceType::Rtc, "rtc".to_string()),
            MmioDeviceInfo {
                addr: addr.0,
-                len: MMIO_LEN,
                irq: rtc_irq,
            },
        );
@@ -1594,7 +1570,6 @@ impl DeviceManager {
            (DeviceType::Gpio, "gpio".to_string()),
            MmioDeviceInfo {
                addr: addr.0,
-                len: MMIO_LEN,
                irq: gpio_irq,
            },
        );
@@ -1698,7 +1673,6 @@ impl DeviceManager {
            (DeviceType::Serial, DeviceType::Serial.to_string()),
            MmioDeviceInfo {
                addr: addr.0,
-                len: MMIO_LEN,
                irq: serial_irq,
            },
        );
@@ -2102,6 +2076,7 @@ impl DeviceManager {
                        net_cfg.num_queues,
                        net_cfg.queue_size,
                        self.seccomp_action.clone(),
+                        net_cfg.rate_limiter_config,
                    )
                    .map_err(DeviceManagerError::CreateVirtioNet)?,
                ))
@@ -2114,6 +2089,7 @@ impl DeviceManager {
                        net_cfg.iommu,
                        net_cfg.queue_size,
                        self.seccomp_action.clone(),
+                        net_cfg.rate_limiter_config,
                    )
                    .map_err(DeviceManagerError::CreateVirtioNet)?,
                ))
@@ -2130,6 +2106,7 @@ impl DeviceManager {
                        net_cfg.num_queues,
                        net_cfg.queue_size,
                        self.seccomp_action.clone(),
+                        net_cfg.rate_limiter_config,
                    )
                    .map_err(DeviceManagerError::CreateVirtioNet)?,
                ))
@@ -3136,11 +3113,9 @@ impl DeviceManager {
    }

    pub fn interrupt_controller(&self) -> Option<Arc<Mutex<dyn InterruptController>>> {
-        if let Some(interrupt_controller) = &self.interrupt_controller {
-            Some(interrupt_controller.clone() as Arc<Mutex<dyn InterruptController>>)
-        } else {
-            None
-        }
+        self.interrupt_controller
+            .as_ref()
+            .map(|ic| ic.clone() as Arc<Mutex<dyn InterruptController>>)
    }

    pub fn console(&self) -> &Arc<Console> {
@@ -3894,32 +3869,18 @@ impl Snapshottable for DeviceManager {
        }

        // Then we store the DeviceManager state.
-        snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", DEVICE_MANAGER_SNAPSHOT_ID),
-            snapshot: serde_json::to_vec(&self.state())
-                .map_err(|e| MigratableError::Snapshot(e.into()))?,
-        });
+        snapshot.add_data_section(SnapshotDataSection::new_from_state(
+            DEVICE_MANAGER_SNAPSHOT_ID,
+            &self.state(),
+        )?);

        Ok(snapshot)
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
        // Let's first restore the DeviceManager.
-        if let Some(device_manager_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", DEVICE_MANAGER_SNAPSHOT_ID))
-        {
-            let device_manager_state = serde_json::from_slice(&device_manager_section.snapshot)
-                .map_err(|e| {
-                    MigratableError::Restore(anyhow!("Could not deserialize DeviceManager {}", e))
-                })?;

-            self.set_state(&device_manager_state);
-        } else {
-            return Err(MigratableError::Restore(anyhow!(
-                "Could not find DeviceManager snapshot section"
-            )));
-        }
+        self.set_state(&snapshot.to_state(DEVICE_MANAGER_SNAPSHOT_ID)?);

        // Now that DeviceManager is updated with the right states, it's time
        // to create the devices based on the configuration.
@@ -3982,6 +3943,12 @@ impl BusDevice for DeviceManager {
                // Clear the PCID bitmap
                self.pci_devices_down = 0;
            }
+            B0EJ_FIELD_OFFSET => {
+                assert!(data.len() == B0EJ_FIELD_SIZE);
+                // Always return an empty bitmap since the eject is always
+                // taken care of right away during a write access.
+                data.copy_from_slice(&[0, 0, 0, 0]);
+            }
            _ => error!(
                "Accessing unknown location at base 0x{:x}, offset 0x{:x}",
                base, offset
--- a/vmm/src/interrupt.rs
+++ b/vmm/src/interrupt.rs
@@ -115,7 +115,7 @@ impl MsiInterruptGroup<IrqRoutingEntry> {
    }
 }

-impl<IrqRoutingEntry> MsiInterruptGroup<IrqRoutingEntry> {
+impl MsiInterruptGroup<IrqRoutingEntry> {
    fn new(
        vm: Arc<dyn hypervisor::Vm>,
        gsi_msi_routes: Arc<Mutex<HashMap<u32, RoutingEntry<IrqRoutingEntry>>>>,
--- a/vmm/src/lib.rs
+++ b/vmm/src/lib.rs
@@ -144,6 +144,10 @@ pub enum Error {
    /// Error creating API server
    #[error("Error creating API server {0:?}")]
    CreateApiServer(micro_http::ServerError),
+
+    /// Error binding API server socket
+    #[error("Error creation API server's socket {0:?}")]
+    CreateApiServerSocket(#[source] io::Error),
 }
 pub type Result<T> = result::Result<T, Error>;

@@ -247,9 +251,11 @@ impl Serialize for PciDeviceInfo {
    }
 }

+#[allow(clippy::too_many_arguments)]
 pub fn start_vmm_thread(
    vmm_version: String,
    http_path: &Option<String>,
+    http_fd: Option<RawFd>,
    api_event: EventFd,
    api_sender: Sender<ApiRequest>,
    api_receiver: Receiver<ApiRequest>,
@@ -280,9 +286,11 @@ pub fn start_vmm_thread(
        })
        .map_err(Error::VmmThreadSpawn)?;

+    // The VMM thread is started, we can start serving HTTP requests
    if let Some(http_path) = http_path {
-        // The VMM thread is started, we can start serving HTTP requests
-        api::start_http_thread(http_path, http_api_event, api_sender, seccomp_action)?;
+        api::start_http_path_thread(http_path, http_api_event, api_sender, seccomp_action)?;
+    } else if let Some(http_fd) = http_fd {
+        api::start_http_fd_thread(http_fd, http_api_event, api_sender, seccomp_action)?;
    }
    Ok(thread)
 }
--- a/vmm/src/memory_manager.rs
+++ b/vmm/src/memory_manager.rs
@@ -488,7 +488,7 @@ impl MemoryManager {
                self.guest_memory
                    .memory()
                    .read_exact_from(
-                        region.start_addr,
+                        GuestAddress(region.start_addr),
                        &mut memory_region_file,
                        region.size as usize,
                    )
@@ -853,47 +853,30 @@ impl MemoryManager {
        if let Some(source_url) = source_url {
            let vm_snapshot_path = url_to_path(source_url).map_err(Error::Restore)?;

-            if let Some(mem_section) = snapshot
-                .snapshot_data
-                .get(&format!("{}-section", MEMORY_MANAGER_SNAPSHOT_ID))
-            {
-                let mem_snapshot: MemoryManagerSnapshotData =
-                    match serde_json::from_slice(&mem_section.snapshot) {
-                        Ok(snapshot) => snapshot,
-                        Err(error) => {
-                            return Err(Error::Restore(MigratableError::Restore(anyhow!(
-                                "Could not deserialize MemoryManager {}",
-                                error
-                            ))))
-                        }
-                    };
+            let mem_snapshot: MemoryManagerSnapshotData = snapshot
+                .to_state(MEMORY_MANAGER_SNAPSHOT_ID)
+                .map_err(Error::Restore)?;

-                // Here we turn the content file name into a content file path as
-                // this will be needed to copy the content of the saved memory
-                // region into the newly created memory region.
-                // We simply ignore the content files that are None, as they
-                // represent regions that have been directly saved by the user, with
-                // no need for saving into a dedicated external file. For these
-                // files, the VmConfig already contains the information on where to
-                // find them.
-                let mut saved_regions = mem_snapshot.memory_regions;
-                for region in saved_regions.iter_mut() {
-                    if let Some(content) = &mut region.content {
-                        let mut memory_region_path = vm_snapshot_path.clone();
-                        memory_region_path.push(content.clone());
-                        *content = memory_region_path;
-                    }
+            // Here we turn the content file name into a content file path as
+            // this will be needed to copy the content of the saved memory
+            // region into the newly created memory region.
+            // We simply ignore the content files that are None, as they
+            // represent regions that have been directly saved by the user, with
+            // no need for saving into a dedicated external file. For these
+            // files, the VmConfig already contains the information on where to
+            // find them.
+            let mut saved_regions = mem_snapshot.memory_regions;
+            for region in saved_regions.iter_mut() {
+                if let Some(content) = &mut region.content {
+                    let mut memory_region_path = vm_snapshot_path.clone();
+                    memory_region_path.push(content.clone());
+                    *content = memory_region_path.to_str().unwrap().to_owned();
                }
-
-                mm.lock().unwrap().fill_saved_regions(saved_regions)?;
-
-                Ok(mm)
-            } else {
-                Err(Error::Restore(MigratableError::Restore(anyhow!(
-                    "Could not find {}-section from snapshot",
-                    MEMORY_MANAGER_SNAPSHOT_ID
-                ))))
            }
+
+            mm.lock().unwrap().fill_saved_regions(saved_regions)?;
+
+            Ok(mm)
        } else {
            Ok(mm)
        }
@@ -1493,25 +1476,13 @@ impl MemoryManager {
        &self.memory_zones
    }

-    // Generate a table for the pages that are dirty. The algorithm is currently
-    // very simple. If any page in a "block" of 64 pages is dirty then that whole
-    // "block" is added to the table. These "blocks" are also collapsed together if
-    // they are contiguous:
-    //
-    // For every block of 64 pages we check to see if there are any dirty pages in it
-    // if there are we increase the size of the current range if there is one, otherwise
-    // create a new range. If there are no dirty pages in the current "block" the range is
-    // closed if there is one and added to the table. After iterating through the dirty
-    // page bitmaps an open range will be closed and added to the table.
-    //
-    // This algorithm is very simple and could be refined to count the number of pages
-    // in the block and instead create smaller ranges covering those pages.
+    // Generate a table for the pages that are dirty. The dirty pages are collapsed
+    // together in the table if they are contiguous.
    pub fn dirty_memory_range_table(
        &self,
    ) -> std::result::Result<MemoryRangeTable, MigratableError> {
        let page_size = 4096; // TODO: Does this need to vary?
        let mut table = MemoryRangeTable::default();
-        let mut total_pages = 0;
        for r in &self.guest_ram_mappings {
            let dirty_bitmap = self.vm.get_dirty_log(r.slot, r.size).map_err(|e| {
                MigratableError::MigrateSend(anyhow!("Error getting VM dirty log {}", e))
@@ -1519,18 +1490,21 @@ impl MemoryManager {

            let mut entry: Option<MemoryRange> = None;
            for (i, block) in dirty_bitmap.iter().enumerate() {
-                if *block > 0 {
-                    if let Some(entry) = &mut entry {
-                        entry.length += 64 * page_size;
-                    } else {
-                        entry = Some(MemoryRange {
-                            gpa: r.gpa + (64 * i as u64 * page_size),
-                            length: 64 * page_size,
-                        });
+                for j in 0..64 {
+                    let is_page_dirty = ((block >> j) & 1u64) != 0u64;
+                    let page_offset = ((i * 64) + j) as u64 * page_size;
+                    if is_page_dirty {
+                        if let Some(entry) = &mut entry {
+                            entry.length += page_size;
+                        } else {
+                            entry = Some(MemoryRange {
+                                gpa: r.gpa + page_offset,
+                                length: page_size,
+                            });
+                        }
+                    } else if let Some(entry) = entry.take() {
+                        table.push(entry);
                    }
-                    total_pages += block.count_ones() as u64;
-                } else if let Some(entry) = entry.take() {
-                    table.push(entry);
                }
            }
            if let Some(entry) = entry.take() {
@@ -1541,18 +1515,9 @@ impl MemoryManager {
                info!("Dirty Memory Range Table is empty");
            } else {
                info!("Dirty Memory Range Table:");
-                let mut total_size = 0;
                for range in table.regions() {
                    info!("GPA: {:x} size: {} (KiB)", range.gpa, range.length / 1024);
-                    total_size += range.length;
                }
-                info!(
-                    "Total pages: {} ({} KiB). Total size: {} KiB. Efficiency: {}%",
-                    total_pages,
-                    total_pages * page_size / 1024,
-                    total_size / 1024,
-                    (total_pages * page_size * 100) / total_size
-                );
            }
        }
        Ok(table)
@@ -1810,9 +1775,17 @@ impl Aml for MemoryMethods {
                        &aml::Path::new("MINH"),
                        &aml::Path::new("LENH"),
                    ),
+                    &aml::If::new(
+                        &aml::LessThan::new(&aml::Path::new("MAXL"), &aml::Path::new("MINL")),
+                        vec![&aml::Add::new(
+                            &aml::Path::new("MAXH"),
+                            &aml::ONE,
+                            &aml::Path::new("MAXH"),
+                        )],
+                    ),
                    &aml::Subtract::new(
-                        &aml::Path::new("MAXH"),
-                        &aml::Path::new("MAXH"),
+                        &aml::Path::new("MAXL"),
+                        &aml::Path::new("MAXL"),
                        &aml::ONE,
                    ),
                    // Release lock
@@ -1955,10 +1928,9 @@ pub struct GuestAddressDef(pub u64);

 #[derive(Clone, Serialize, Deserialize)]
 pub struct MemoryRegion {
-    content: Option<PathBuf>,
-    #[serde(with = "GuestAddressDef")]
-    start_addr: GuestAddress,
-    size: GuestUsize,
+    content: Option<String>,
+    start_addr: u64,
+    size: u64,
 }

 #[derive(Serialize, Deserialize)]
@@ -2001,8 +1973,8 @@ impl Snapshottable for MemoryManager {
            }

            memory_regions.push(MemoryRegion {
-                content,
-                start_addr: region.start_addr(),
+                content: content.map(|p| p.to_str().unwrap().to_owned()),
+                start_addr: region.start_addr().0,
                size: region.len(),
            });

@@ -2019,14 +1991,10 @@ impl Snapshottable for MemoryManager {
        // memory region content for the regions requiring it.
        self.snapshot_memory_regions = memory_regions.clone();

-        let snapshot_data_section =
-            serde_json::to_vec(&MemoryManagerSnapshotData { memory_regions })
-                .map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        memory_manager_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", MEMORY_MANAGER_SNAPSHOT_ID),
-            snapshot: snapshot_data_section,
-        });
+        memory_manager_snapshot.add_data_section(SnapshotDataSection::new_from_state(
+            MEMORY_MANAGER_SNAPSHOT_ID,
+            &MemoryManagerSnapshotData { memory_regions },
+        )?);

        let mut memory_snapshot = self.snapshot.lock().unwrap();
        *memory_snapshot = Some(guest_memory);
@@ -2059,7 +2027,7 @@ impl Transportable for MemoryManager {

                    guest_memory
                        .write_all_to(
-                            region.start_addr,
+                            GuestAddress(region.start_addr),
                            &mut memory_region_file,
                            region.size as usize,
                        )
--- a/vmm/src/seccomp_filters.rs
+++ b/vmm/src/seccomp_filters.rs
@@ -344,6 +344,7 @@ fn vmm_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
        allow_syscall(libc::SYS_mremap),
        allow_syscall(libc::SYS_munmap),
        allow_syscall(libc::SYS_nanosleep),
+        allow_syscall(libc::SYS_newfstatat),
        #[cfg(target_arch = "x86_64")]
        allow_syscall(libc::SYS_open),
        allow_syscall(libc::SYS_openat),
@@ -418,6 +419,7 @@ fn vcpu_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
    Ok(vec![
        allow_syscall(libc::SYS_brk),
        allow_syscall(libc::SYS_clock_gettime),
+        allow_syscall(libc::SYS_clock_nanosleep),
        allow_syscall(libc::SYS_close),
        allow_syscall(libc::SYS_dup),
        allow_syscall(libc::SYS_exit),
@@ -431,6 +433,7 @@ fn vcpu_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
        allow_syscall(libc::SYS_mprotect),
        allow_syscall(libc::SYS_munmap),
        allow_syscall(libc::SYS_nanosleep),
+        allow_syscall(libc::SYS_newfstatat),
        #[cfg(target_arch = "x86_64")]
        allow_syscall(libc::SYS_open),
        allow_syscall(libc::SYS_openat),
@@ -458,7 +461,6 @@ fn vcpu_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
 fn api_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
    Ok(vec![
        allow_syscall(libc::SYS_accept4),
-        allow_syscall(libc::SYS_bind),
        allow_syscall(libc::SYS_brk),
        allow_syscall(libc::SYS_close),
        allow_syscall(libc::SYS_dup),
@@ -471,13 +473,11 @@ fn api_thread_rules() -> Result<Vec<SyscallRuleSet>, Error> {
        allow_syscall(libc::SYS_futex),
        allow_syscall(libc::SYS_getrandom),
        allow_syscall_if(libc::SYS_ioctl, create_api_ioctl_seccomp_rule()?),
-        allow_syscall(libc::SYS_listen),
        allow_syscall(libc::SYS_madvise),
        allow_syscall(libc::SYS_mprotect),
        allow_syscall(libc::SYS_munmap),
        allow_syscall(libc::SYS_recvfrom),
        allow_syscall(libc::SYS_sigaltstack),
-        allow_syscall(libc::SYS_socket),
        allow_syscall(libc::SYS_write),
    ])
 }
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -2422,29 +2422,11 @@ mod tests {
    use arch::aarch64::fdt::create_fdt;
    use arch::aarch64::gic::kvm::create_gic;
    use arch::aarch64::{layout, DeviceInfoForFdt};
-    use arch::DeviceType;
+    use arch::{DeviceType, MmioDeviceInfo};
    use vm_memory::{GuestAddress, GuestMemoryMmap};

    const LEN: u64 = 4096;

-    #[derive(Clone, Debug)]
-    pub struct MmioDeviceInfo {
-        addr: u64,
-        irq: u32,
-    }
-
-    impl DeviceInfoForFdt for MmioDeviceInfo {
-        fn addr(&self) -> u64 {
-            self.addr
-        }
-        fn irq(&self) -> u32 {
-            self.irq
-        }
-        fn length(&self) -> u64 {
-            LEN
-        }
-    }
-
    #[test]
    fn test_create_fdt_with_devices() {
        let mut regions = Vec::new();